diff --git a/.hgignore b/.hgignore index 4e3c3f1..17b7297 100644 --- a/.hgignore +++ b/.hgignore @@ -1,2 +1,3 @@ syntax: glob -bin \ No newline at end of file +bin +cover.out \ No newline at end of file diff --git a/.vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a b/.vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a new file mode 100644 index 0000000..6d88734 Binary files /dev/null and b/.vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a differ diff --git a/Makefile b/Makefile index 9c4855c..04b7e24 100644 --- a/Makefile +++ b/Makefile @@ -3,18 +3,20 @@ # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/ # build targets (which aren't files) -.PHONY: build fmt lint vet test autotest run clean env vendor +.PHONY: build fmt lint vet test autotest cover doc run clean env stats vendor # configuration APPNAME := docrawler VENDOR := .vendor PACKAGES := \ + golang.org/x/tools/cmd/cover \ + golang.org/x/tools/cmd/godoc \ golang.org/x/tools/cmd/vet \ github.com/golang/lint/golint # override GOPATH to be our vendor directory -export GOPATH := ${PWD}/${VENDOR} -export PATH := ${PWD}/${VENDOR}/bin:${PATH} +export GOPATH := ${PWD}/${VENDOR}:${PWD} +export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH} default: build @@ -24,35 +26,54 @@ build: fmt lint vet fmt: go fmt ./src/... +# TODO golint is not accepting the "./src/..." for some reason (from within make) lint: - golint ./src + golint ./src/${APPNAME}/*.go vet: go vet ./src/... test: - go test ./src/... | ./testfilter.sh + go test -v ./src/... | ./util/testfilter.sh autotest: fswatch -or ./src | \ - xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./testfilter.sh' + xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh' + +cover: + go test -coverprofile cover.out ./src/... && go tool cover -html cover.out + +doc: + godoc -http=:6060 -index run: build ./bin/${APPNAME} clean: - rm -f ./bin/${APPNAME} + rm -f ./bin/${APPNAME} ./cover.out env: go env +stats: + @echo =============================================================================== + @echo Tests + @echo =============================================================================== + @cloc --match-f=_test.go ./src + @echo + @echo + @echo =============================================================================== + @echo Everything Else + @echo =============================================================================== + @cloc --not-match-f=_test.go ./src + # vendor # * wipes our vendor directory completely -# * fetches all packages in ${PACKAGES} +# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only) # * removes source control dirs so packages are vendorable (commitable) vendor: - rm -dRf ./${VENDOR}/src && \ - go get -u ${PACKAGES} && \ + rm -dRf ./${VENDOR} && \ + GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES} && \ rm -rf `find ./${VENDOR}/src -type d -name .git` && \ rm -rf `find ./${VENDOR}/src -type d -name .hg` && \ rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \ diff --git a/docs/Design.md b/docs/Design.md index 183f389..db2d874 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -39,11 +39,11 @@ These are features I'd like to include. * use channels to form a pipeline * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines) +* pipeline may allows us to scale concurrency at the bottlenecks * could use Interfaces for things like Parser, Getter, but I think best avoided until needed * launch goroutine per fetch * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it - ## Wishlist These are features that sound cool but are probably out of scope. diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 0000000..3e03d44 --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,22 @@ +# To Do + +## Bugs + +* deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.) + * should only crawl these pages once + * but should be included in the links part +* when sorting links, "uniq" them as well + +## Cleanup + +* rename Page -> crawlerState +* helper function to do the whole crawl process +* document code better + * test with godoc + +## Features + +* some sort of status during crawl? +* command line options, flag parsing +* check RFCs on URL character sets, compliance, etc. +* packagize parts of app diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index c8f4ae0..48de706 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,6 +1,93 @@ package main -import () +import ( + "fmt" + "os" +) + +// docrawl begins crawling the site at "url" +func docrawl(url string) []*Page { + // our result structure + var sitemap []*Page + + // what we've already parsed, what we still need to parse + crawled := make(map[string]*Page) + queued := make(map[string]*Page) + + // create first page and add to the queue + homepage, _ := NewPage(nil, url) // TODO check error + queued[homepage.URL.String()] = homepage + + // main loop + for len(queued) > 0 { + // pop a page off of queued + var dest *Page + var destURL string + for destURL, dest = range queued { + break + } + delete(queued, destURL) + + // mark this page as complete + crawled[dest.URL.String()] = dest + + // add page to list + sitemap = append(sitemap, dest) + + // make sure this URL has the same hostname as our first page + if dest.URL.Host != homepage.URL.Host { + // skip URLs associated with other Hosts + dest.Skipped = true + continue + } + + // fetch page + text, err := fetchPage(dest) + if err != nil { + dest.Broken = true + continue + } + + // parse links + title, links, err := parseLinks(text) + if err != nil { + dest.Broken = true + continue + } + dest.Title = title + for _, l := range links { + p, err := NewPage(dest, l) + if err != nil { + continue // TODO check error + } + + // see if we already know about this page + pCrawled, haveCrawled := crawled[p.URL.String()] + pQueued, haveQueued := queued[p.URL.String()] + if haveCrawled { + // add the previously crawled page to children + dest.Children = append(dest.Children, pCrawled) + } else if haveQueued { + // add the previously queued page to children + dest.Children = append(dest.Children, pQueued) + } else { + // add this new page (never seen) to children + dest.Children = append(dest.Children, p) + // .. and queue it up for crawling + queued[p.URL.String()] = p + } + } + } + + return sitemap +} func main() { + // crawl each URL on the command line + for _, u := range os.Args[1:] { + pages := docrawl(u) + l := sitemapToLocations(pages) + j, _ := locationsToJSON(l) + fmt.Println(j) + } } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index a2c50c7..6233cac 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net" "net/http" "os" @@ -19,13 +18,13 @@ func TestMain(m *testing.M) { // start our simple web server l, err := net.Listen("tcp", ":"+strconv.Itoa(port)) if err != nil { - panic("unable to open port for http server") // TODO remove panic? + panic("unable to open port for http server") } http.Handle("/", http.FileServer(http.Dir("./testsite"))) go func() { err = http.Serve(l, nil) if err != nil { - panic("unable to start http server") // TODO remove panic? + panic("unable to start http server") } }() @@ -43,8 +42,8 @@ func TestMain(m *testing.M) { func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error(err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) + t.Fatal(err) } } @@ -52,46 +51,191 @@ func TestServerRunning(t *testing.T) { func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error("request for non existent file didn't fail properly") + t.Logf("got %v, err = %v\n", resp.StatusCode, err) + t.Fatal("request for non existent file didn't fail properly") } } -// TestFetch makes sure we can fetch a file and we get exactly what we expect -func TestFetch(t *testing.T) { - body, err := fetch(baseURL + "fetch_test.html") +// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect +func TestSimpleFetch(t *testing.T) { + page, err := NewPage(nil, baseURL+"fetch_test.html") if err != nil { - t.Error(err) + t.Error("problem creating New Page struct") + } + body, err := fetchPage(page) + if err != nil { + t.Fatal(err) } desired := "\n" if body != desired { - fmt.Printf(" Got: '%v'\n", body) - fmt.Printf("Wanted: '%v'\n", desired) + t.Logf(" Got: '%v'\n", body) + t.Logf("Wanted: '%v'\n", desired) t.Error("fetch succeeded, but content mismatched") } } -// TestParse verifies that we can extract URLs (etc.) from a known document -func TestParse(t *testing.T) { - doc := ` - - - - - - -` - matches, err := parse(doc) +// TestSimpleMap figures out the site map for the site in baseURL +func TestSimpleMap(t *testing.T) { + pages := docrawl(baseURL) + + // because our crawl is non-deterministic, we have to do a complete + // cycle through every page, counting stuff, finding specific pages + // so that we can then verify certain aspects of each page + skipped, children := 0, 0 + var homepage, aboutpage, skippage *Page + for _, p := range pages { + if p.Title == "Home" { + homepage = p + } else if p.Title == "About Test" { + aboutpage = p + } else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" { + skippage = p + } + if p.Skipped { + skipped++ + } + children += len(p.Children) + } + + // check page counts + if len(pages) != 6 { + t.Logf("got %v, wanted %v\n", len(pages), 6) + t.Fatal("got wrong number of pages") + } + if children != 8 { + t.Logf("got %v, wanted %v\n", children, 8) + t.Fatal("got wrong number of total children") + } + if skipped != 1 { + t.Logf("got %v, wanted %v\n", skipped, 1) + t.Fatal("got wrong number of total skipped") + } + + // check first page URL + if homepage.URL.String() != baseURL { + t.Error("page URL is invalid") + } + + // specific page number of children + if len(homepage.Children) != 4 { + t.Logf("got %v, wanted %v\n", len(homepage.Children), 4) + t.Fatal("got wrong number of links") + } + if len(aboutpage.Children) != 4 { + t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4) + t.Fatal("got wrong number of links") + } + + // make sure remote pages were skipped + if homepage.Skipped != false { + t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String()) + t.Error("child was skipped") + } + if skippage.Skipped != true { + t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String()) + t.Error("child wasn't skipped") + } + + // verify page content + // TODO find a better way to do this + //if homepage.Children[0].URL.String() != baseURL+"about.html" { + //t.Logf(" Got: %q\n", homepage.Children[0].URL.String()) + //t.Logf("Wanted: %q\n", baseURL+"about.html") + //t.Error("link name is incorrect") + //} + //if homepage.Children[1].URL.String() != baseURL+"assets/image.png" { + //t.Logf(" Got: %q\n", homepage.Children[1].URL.String()) + //t.Logf("Wanted: %q\n", baseURL+"assets/image.png") + //t.Error("link name is incorrect") + //} + //if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" { + //t.Logf(" Got: %q\n", homepage.Children[3].URL.String()) + //t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") + //t.Error("link name is incorrect") + //} +} + +// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results +func TestHeaderFetching(t *testing.T) { + basepage, err := NewPage(nil, baseURL) if err != nil { - t.Error(err) + t.Error("problem creating New Page struct") } - if len(matches) != 2 { - t.Error("invalid number of matches in parse") + if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { + t.Error("problem fetching filetype") } - if matches[0] != "/assets/image.png" { - t.Error("match text is invalid") + + page, err := NewPage(basepage, "about.html") + if err != nil { + t.Error("problem creating New Page struct") } - if matches[1] != "/about.html" { - t.Error("match text is invalid") + if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { + t.Error("problem fetching filetype") + } + + page, err = NewPage(basepage, "assets/image.png") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { + t.Logf("got %q, wanted %q", page.MediaType, "image/png") + t.Error("problem fetching filetype") } } + +// TestJsonOutput gets a sitemap and then converts it to json +func TestJsonOutput(t *testing.T) { + pages := docrawl(baseURL) + l := sitemapToLocations(pages) + if len(l) != 2 { + t.Error("sitemapToLocations has the wrong number of locations") + } + if l[0].URL != baseURL { + t.Error("got an incorrect location") + } + if len(l[0].Links) != 1 { + t.Error("location 0 has wrong number of links") + } + if len(l[0].Assets) != 2 { + t.Logf("got %v", len(l[0].Assets)) + t.Error("location 0 has wrong number of assets") + } + if len(l[0].Broken) != 1 { + t.Logf("got %v", len(l[0].Broken)) + t.Error("location 0 has wrong number of broken urls") + } + if len(l[0].Remote) != 0 { + t.Logf("got %v", len(l[1].Remote)) + t.Error("location 0 has wrong number of remote urls") + } + if l[1].URL != baseURL+"about.html" { + t.Error("got an incorrect location") + } + if len(l[1].Links) != 1 { + t.Error("location 1 has wrong number of links") + } + if len(l[1].Assets) != 2 { + t.Error("location 1 has wrong number of assets") + } + if len(l[1].Broken) != 0 { + t.Logf("got %v", len(l[1].Broken)) + t.Error("location 1 has wrong number of broken urls") + } + if len(l[1].Remote) != 1 { + t.Logf("got %v", len(l[1].Remote)) + t.Error("location 1 has wrong number of remote urls") + } + + // this is a cheater test because the output is from a run of the code being + // test itself. but, it's been examined and I think it's right. and, there's not + // many other ways to test this without some significant pain + cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]` + j, err := locationsToJSON(l) + if err != nil { + t.Error("locationsToJSON failed") + } + if j != cheaterTest { + t.Error("locationsToJSON text doesn't match") + } + t.Log(j) +} diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index bdf1836..9d41dc9 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -1,22 +1,79 @@ package main import ( + "errors" "io/ioutil" + "mime" "net/http" ) -func fetch(url string) (string, error) { - // GET the url - resp, err := http.Get(url) +var ( + errContentTypeNotFound = errors.New("no Content-Type header found") + errFetchError = errors.New("couldn't fetch page") +) + +// fetchFiletype performs an http HEAD to get the media type, and sets it +// directly in Page.MediaType +func fetchFiletype(page *Page) error { + resp, err := http.Head(page.URL.String()) + if err != nil { + return err + } + + // check response code + if resp.StatusCode != http.StatusOK { + return errFetchError + } + + // pull the content type out of the http header + contentType, ok := resp.Header["Content-Type"] + if !ok { + // no Content-Type header found + return errContentTypeNotFound + } + + // parse mime type + mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return err + } + + // set and return + page.MediaType = mediatype + return nil +} + +// fetchPage takes a Page, GETs it, and returns the body as a string +func fetchPage(page *Page) (string, error) { + // figure out the file type + err := fetchFiletype(page) if err != nil { return "", err } + // we only want to fetch html + if page.MediaType != "text/html" { + return "", nil // but this isn't an error! + } + + // GET the url + resp, err := http.Get(page.URL.String()) + if err != nil { + return "", err + } + + // check response code + if resp.StatusCode != http.StatusOK { + return "", errFetchError + } + // read the body body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } + + // success! close the body, return it as a string resp.Body.Close() return string(body), nil } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go new file mode 100644 index 0000000..7cdff84 --- /dev/null +++ b/src/docrawler/fetch_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" +) + +// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures +func TestFailedHeaderFetching(t *testing.T) { + page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err == nil { + t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype") + } +} + +// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures +func TestFailedFetching(t *testing.T) { + page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + if err != nil { + t.Error("problem creating New Page struct") + } + if _, err := fetchPage(page); err == nil { + t.Error("tired fetching bogus page but didn't get nil back from fetchPage") + } +} diff --git a/src/docrawler/output.go b/src/docrawler/output.go new file mode 100644 index 0000000..87f5ad1 --- /dev/null +++ b/src/docrawler/output.go @@ -0,0 +1,67 @@ +package main + +import ( + "encoding/json" + "sort" +) + +// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc. +type Location struct { + URL string + Title string + Links []string + Assets []string + Broken []string + Remote []string +} + +func sitemapToLocations(pages []*Page) []*Location { + // build a map of our pages + pageMap := make(map[string]*Page) + for _, p := range pages { + pageMap[p.URL.String()] = p + } + + // build a slice of locations (one per page) + var locations []*Location + for _, p := range pages { + if p.MediaType == "text/html" { + // create a location for this page + l := &Location{URL: p.URL.String(), Title: p.Title} + + // add its children + for _, c := range p.Children { + // look up this child's media type from the root list of pages + //mediaType := pageMap[c.URL.String()].MediaType + if c.Skipped { + l.Remote = append(l.Remote, c.URL.String()) + } else if c.MediaType == "text/html" { + l.Links = append(l.Links, c.URL.String()) + } else if c.Broken { + l.Broken = append(l.Broken, c.URL.String()) + } else { + l.Assets = append(l.Assets, c.URL.String()) + } + } + + // now sort the children slices + sort.Strings(l.Remote) + sort.Strings(l.Links) + sort.Strings(l.Broken) + sort.Strings(l.Assets) + + // and add this location to our slice + locations = append(locations, l) + } + } + + return locations +} + +func locationsToJSON(locations []*Location) (string, error) { + b, err := json.Marshal(locations) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/src/docrawler/page.go b/src/docrawler/page.go new file mode 100644 index 0000000..9e8be32 --- /dev/null +++ b/src/docrawler/page.go @@ -0,0 +1,33 @@ +package main + +import ( + "net/url" +) + +// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type Page struct { + URL *url.URL + Title string + MediaType string + Skipped bool + Broken bool + Children []*Page +} + +// NewPage takes a referring Page + a URL and returns a new &Page{} +func NewPage(referrer *Page, rawurl string) (*Page, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.URL.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // create struct and return + return &Page{URL: u, Skipped: false, Broken: false}, nil +} diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go new file mode 100644 index 0000000..217f290 --- /dev/null +++ b/src/docrawler/page_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" +) + +// TestBadNewPage tests NewPage to see if we are getting expected failures +func TestBadNewPage(t *testing.T) { + _, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") + if err == nil { + t.Error("tried creating a NewPage with an invalid URL but didn't get an error") + } +} diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 483dbdb..1c735d6 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -2,16 +2,28 @@ package main import ( "regexp" + "strings" ) -func parse(s string) ([]string, error) { - // hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.) - re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) +// a regexp which captures src/href/xhref="..." from text +var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant +var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) - // find all of the matches (submatch = include capture groups) +// parseLinks takes a string and attempts to parse any html title and all links out of it, +// and returns a slice of the captures found +func parseLinks(s string) (string, []string, error) { + // find the title, or default to empty + title := "" + titleMatches := reTitle.FindStringSubmatch(s) + if titleMatches != nil { + // we found a title, replace our default title with the result + title = strings.TrimSpace(titleMatches[1]) + } + + // find all of the URL matches (submatch = include capture groups) results := []string{} - for _, match := range re.FindAllStringSubmatch(s, -1) { + for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } - return results, nil + return title, results, nil } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go new file mode 100644 index 0000000..a04b869 --- /dev/null +++ b/src/docrawler/parse_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "testing" +) + +// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document +func TestSimpleParse(t *testing.T) { + doc := ` + + Test Page + + + + fooey! +