From 72cac47961901c1d8cefe0efbc069e84c37b66e6 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:13:52 -0800 Subject: [PATCH] docrawl() completely refactored to use goroutine per URL * outputs status * crawlStatus() function created (split out of docrawl()) * changed from using bool flags to linkType in single httpItem enum, tests updated to match this * unified where we set httpItem.linkType (inside fetchType), appropriate errors and returns added * renamed a lot of things from 'page' to 'item' * include refurl in httpItem * []*Location result is now sorted --- docs/TODO.md | 2 - src/docrawler/docrawler.go | 158 +++++++++++++++++++------------- src/docrawler/docrawler_test.go | 14 +-- src/docrawler/fetch.go | 34 +++++-- src/docrawler/item.go | 36 ++++++-- src/docrawler/output.go | 21 ++++- 6 files changed, 171 insertions(+), 94 deletions(-) diff --git a/docs/TODO.md b/docs/TODO.md index bcf97f6..52dea6e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -10,14 +10,12 @@ ## Cleanup -* rename Page -> crawlerState * helper function to do the whole crawl process * document code better * test with godoc ## Features -* some sort of status during crawl? * command line options, flag parsing * check RFCs on URL character sets, compliance, etc. * packagize parts of app diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 70e8702..af538a3 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,88 +2,120 @@ package main import ( "fmt" + "log" "os" + "time" ) -type itemSlice []*httpItem -type itemMap map[string]*httpItem +// docrawl begins crawling the site at "homeurl" +func docrawl(homeurl string) itemSlice { + // set of what we have already crawled + crawled := make(map[string]struct{}) + // map (of urlString -> httpItem) of our results + results := make(itemMap) -// docrawl begins crawling the site at "url" -func docrawl(url string) itemSlice { - // our result structure - var sitemap itemSlice + // a channel to receive our results on + rchan := make(chan *httpItem) - // what we've already parsed, what we still need to parse - crawled := make(itemMap) - queued := make(itemMap) + // start a ticker which we'll use to output status and check for completion + ticker := time.Tick(250 * time.Millisecond) - // create first page and add to the queue - homepage, _ := newHTTPItem(nil, url) // TODO check error - queued[homepage.url.String()] = homepage + // create first page's httpItem + homeitem, err := newHTTPItem(nil, homeurl) + if err != nil { + log.Fatal("unable to create httpItem for homeurl") + } - // main loop - for len(queued) > 0 { - // pop a page off of queued - var dest *httpItem - var destURL string - for destURL, dest = range queued { - break - } - delete(queued, destURL) + // set our number of outstanding pages (initially 1 to account for first page) + crawlingCount := 1 - // mark this page as complete - crawled[dest.url.String()] = dest + // start the home page crawl + crawled[homeitem.url.String()] = struct{}{} + go crawlItem(homeitem, rchan) - // add page to list - sitemap = append(sitemap, dest) + // wait for results + for { + select { + case r := <-rchan: // new results? + // add result to our results map + results[r.url.String()] = r - // make sure this URL has the same hostname as our first page - if dest.url.Host != homepage.url.Host { - // skip URLs associated with other Hosts - dest.skipped = true - continue - } + // decrease the outstanding page count by 1 + crawlingCount-- - // fetch page - text, err := fetchPage(dest) - if err != nil { - dest.broken = true - continue - } + // start crawly any new child pages we haven't yet crawled + for i, c := range r.children { + // see if we already have a result for this page, if so, point to that result + if existing, ok := results[c.url.String()]; ok { + r.children[i] = existing + continue + } - // parse links - title, links, err := parseLinks(text) - if err != nil { - dest.broken = true - continue - } - dest.title = title - for _, l := range links { - p, err := newHTTPItem(dest, l) - if err != nil { - continue // TODO check error + // see if we're already crawling this page (but maybe don't have results yet) + if _, ok := crawled[c.url.String()]; !ok { + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = struct{}{} + go crawlItem(c, rchan) + } } - // see if we already know about this page - pCrawled, haveCrawled := crawled[p.url.String()] - pQueued, haveQueued := queued[p.url.String()] - if haveCrawled { - // add the previously crawled page to children - dest.children = append(dest.children, pCrawled) - } else if haveQueued { - // add the previously queued page to children - dest.children = append(dest.children, pQueued) - } else { - // add this new page (never seen) to children - dest.children = append(dest.children, p) - // .. and queue it up for crawling - queued[p.url.String()] = p + case <-ticker: // our regular ticker. for status output and checking for completion. + // output status to console + log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount) + + // see if we're finished + if crawlingCount == 0 { + // finished! convert results map to a slice and return it + rslice := itemSlice{} + for _, v := range results { + rslice = append(rslice, v) + } + return rslice } } } +} - return sitemap +// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, +// and filling out its structure as much as possible +func crawlItem(item *httpItem, rchan chan<- *httpItem) { + // make sure this item is the same domain (i.e. URL "host part") as its referrer + if item.refurl != nil && item.url.Host != item.refurl.Host { + // skip URLs associated with other Hosts + item.linkType = tRemote + rchan <- item + return + } + + // fetch page + text, err := fetchPage(item) + if err != nil { + rchan <- item + return + } + + // parse links + title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed + if err != nil { + item.linkType = tBroken + rchan <- item + return + } + item.title = title + + // walk links and add them as children to the current item + for _, l := range links { + newItem, err := newHTTPItem(item, l) + if err != nil { + continue // TODO bad item + } + item.children = append(item.children, newItem) + } + + // send back our item struct now that it's all filled out + rchan <- item } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index abd294f..6be6209 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -91,7 +91,7 @@ func TestSimpleMap(t *testing.T) { } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.skipped { + if p.linkType == tRemote { skipped++ } children += len(p.children) @@ -127,11 +127,11 @@ func TestSimpleMap(t *testing.T) { } // make sure remote pages were skipped - if homepage.skipped != false { + if homepage.linkType == tRemote { t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.skipped != true { + if skippage.linkType != tRemote { t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } @@ -161,7 +161,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -169,7 +169,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -177,8 +177,8 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" { - t.Logf("got %q, wanted %q", page.mediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.linkType != tAsset { + t.Logf("got %q, wanted %q", page.linkType, tAsset) t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index c7c83d9..9cd67a6 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -9,19 +9,22 @@ import ( var ( errContentTypeNotFound = errors.New("no Content-Type header found") - errFetchError = errors.New("couldn't fetch page") + errFetchError = errors.New("couldn't fetch item") + errFileTypeUnknown = errors.New("couldn't determine file type") ) // fetchFiletype performs an http HEAD to get the media type, and sets it // directly in httpItem.mediaType -func fetchFiletype(page *httpItem) error { - resp, err := http.Head(page.url.String()) +func fetchFiletype(item *httpItem) error { + resp, err := http.Head(item.url.String()) if err != nil { + item.linkType = tBroken return err } // check response code if resp.StatusCode != http.StatusOK { + item.linkType = tBroken return errFetchError } @@ -29,35 +32,46 @@ func fetchFiletype(page *httpItem) error { contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found + item.linkType = tBroken return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) if err != nil { + item.linkType = tBroken return err } - // set and return - page.mediaType = mediatype + // success, set item type and return + if mediatype == "text/html" { + item.linkType = tHTMLPage + } else { + item.linkType = tAsset + } return nil } // fetchPage takes a httpItem, GETs it, and returns the body as a string -func fetchPage(page *httpItem) (string, error) { +func fetchPage(item *httpItem) (string, error) { // figure out the file type - err := fetchFiletype(page) + err := fetchFiletype(item) if err != nil { return "", err } - // we only want to fetch html - if page.mediaType != "text/html" { + // make sure we got a file type! + if item.linkType == tUnknown { + return "", errFileTypeUnknown + } + + // we only want to fetch html, so return if it's anything else + if item.linkType != tHTMLPage { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.url.String()) + resp, err := http.Get(item.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/item.go b/src/docrawler/item.go index d6f8ac8..cb9d90c 100644 --- a/src/docrawler/item.go +++ b/src/docrawler/item.go @@ -4,14 +4,30 @@ import ( "net/url" ) +// itemSlice is a convenience type for a slice of items +type itemSlice []*httpItem + +// itemMap convenience type for map of string's (URL) to items +type itemMap map[string]*httpItem + +// itemType is an enum so we know how to classify each item +type itemType int + +const ( + tUnknown itemType = iota + tHTMLPage + tAsset + tRemote + tBroken +) + // httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. type httpItem struct { - url *url.URL - title string - mediaType string - skipped bool - broken bool - children []*httpItem + url *url.URL + refurl *url.URL + title string + linkType itemType + children itemSlice } // newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} @@ -28,6 +44,12 @@ func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { return nil, err } + // figure out referrer url + var rurl *url.URL + if referrer != nil { + rurl = referrer.url + } + // create struct and return - return &httpItem{url: u, skipped: false, broken: false}, nil + return &httpItem{url: u, refurl: rurl}, nil } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 5b26247..01012de 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,7 +15,16 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*httpItem) []*Location { +// implement Location slice sorting (by URL) +type byURL []*Location + +// implement Sort interface on byURL (which is a []*Location) +func (l byURL) Len() int { return len(l) } +func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL } +func (l byURL) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling +func sitemapToLocations(pages itemSlice) []*Location { // build a map of our pages pageMap := make(map[string]*httpItem) for _, p := range pages { @@ -25,7 +34,7 @@ func sitemapToLocations(pages []*httpItem) []*Location { // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.mediaType == "text/html" { + if p.linkType == tHTMLPage { // create a location for this page l := &Location{URL: p.url.String(), Title: p.title} @@ -33,11 +42,11 @@ func sitemapToLocations(pages []*httpItem) []*Location { for _, c := range p.children { // look up this child's media type from the root list of pages //mediaType := pageMap[c.url.String()].mediaType - if c.skipped { + if c.linkType == tRemote { l.Remote = append(l.Remote, c.url.String()) - } else if c.mediaType == "text/html" { + } else if c.linkType == tHTMLPage { l.Links = append(l.Links, c.url.String()) - } else if c.broken { + } else if c.linkType == tBroken { l.Broken = append(l.Broken, c.url.String()) } else { l.Assets = append(l.Assets, c.url.String()) @@ -55,6 +64,8 @@ func sitemapToLocations(pages []*httpItem) []*Location { } } + // sort the Locations themselves and return + sort.Sort(byURL(locations)) return locations }