From d9009cdbaac75ff9c1aaff0df54e164fe7830bf1 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 15:50:30 -0800 Subject: [PATCH 01/22] flow: Created branch 'feature/unique_links_from_each_page'. From 2e997084171e4e8abfc2b11b576752088bbacea2 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 16:26:16 -0800 Subject: [PATCH 02/22] won't recrawl pages that are the same after stripping #anchor (blah.html#about, blah.html#contact, etc.) * but broken * committing on feature branch and will come back to this --- docs/TODO.md | 1 + src/docrawler/docrawler.go | 10 +++++----- src/docrawler/output.go | 6 +++++- src/docrawler/page.go | 16 +++++++++------- src/docrawler/url.go | 9 +++++++++ src/docrawler/url_test.go | 11 +++++++++++ src/docrawler/util.go | 23 +++++++++++++++++++++++ src/docrawler/util_test.go | 25 +++++++++++++++++++++++++ util/testfilter.sh | 2 +- 9 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 src/docrawler/util.go create mode 100644 src/docrawler/util_test.go diff --git a/docs/TODO.md b/docs/TODO.md index 3e03d44..450c69e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -13,6 +13,7 @@ * helper function to do the whole crawl process * document code better * test with godoc +* omit fields in json when nil? (Broken, etc.) ## Features diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 48de706..08ac36f 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -16,7 +16,7 @@ func docrawl(url string) []*Page { // create first page and add to the queue homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.URL.String()] = homepage + queued[homepage.Identifier] = homepage // main loop for len(queued) > 0 { @@ -29,7 +29,7 @@ func docrawl(url string) []*Page { delete(queued, destURL) // mark this page as complete - crawled[dest.URL.String()] = dest + crawled[dest.Identifier] = dest // add page to list sitemap = append(sitemap, dest) @@ -62,8 +62,8 @@ func docrawl(url string) []*Page { } // see if we already know about this page - pCrawled, haveCrawled := crawled[p.URL.String()] - pQueued, haveQueued := queued[p.URL.String()] + pCrawled, haveCrawled := crawled[p.Identifier] + pQueued, haveQueued := queued[p.Identifier] if haveCrawled { // add the previously crawled page to children dest.Children = append(dest.Children, pCrawled) @@ -74,7 +74,7 @@ func docrawl(url string) []*Page { // add this new page (never seen) to children dest.Children = append(dest.Children, p) // .. and queue it up for crawling - queued[p.URL.String()] = p + queued[p.Identifier] = p } } } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 87f5ad1..2b6b959 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -44,7 +44,11 @@ func sitemapToLocations(pages []*Page) []*Location { } } - // now sort the children slices + // now uniq & sort the children slices + l.Remote = uniqStrings(l.Remote) + l.Links = uniqStrings(l.Links) + l.Broken = uniqStrings(l.Broken) + l.Assets = uniqStrings(l.Assets) sort.Strings(l.Remote) sort.Strings(l.Links) sort.Strings(l.Broken) diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 9e8be32..e0dcfcb 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -6,12 +6,13 @@ import ( // Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { - URL *url.URL - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page + URL *url.URL + Identifier string + Title string + MediaType string + Skipped bool + Broken bool + Children []*Page } // NewPage takes a referring Page + a URL and returns a new &Page{} @@ -29,5 +30,6 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) { } // create struct and return - return &Page{URL: u, Skipped: false, Broken: false}, nil + p := &Page{URL: u, Skipped: false, Broken: false, Identifier: stripAnchorFromURL(u)} + return p, nil } diff --git a/src/docrawler/url.go b/src/docrawler/url.go index d66bc44..d98657a 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -49,3 +49,12 @@ func resolveURL(referringURL string, currentURL string) (*url.URL, error) { // return the URL we ended up with, and the error from checkURL return uResolved, checkURL(uResolved) } + +func stripAnchorFromURL(u *url.URL) string { + // copy the url locally + ucopy := *u + + // blank out the anchor, return URI string + ucopy.Fragment = "" + return ucopy.String() +} diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index b0867e8..07a4a9b 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -68,3 +68,14 @@ func TestBadResolve(t* testing.T) { } } */ + +func TestURLStripAnchor(t *testing.T) { + u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo") + if err != nil { + t.Fatal("couldn't parse a URL") + } + stripped := stripAnchorFromURL(u) + if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" { + t.Error("stripping anchor #foo from URL failed") + } +} diff --git a/src/docrawler/util.go b/src/docrawler/util.go new file mode 100644 index 0000000..d28531b --- /dev/null +++ b/src/docrawler/util.go @@ -0,0 +1,23 @@ +package main + +// uniqStrings takes a slice of strings and removes any duplicates +// note: does not guarantee any order (or stability of order) +func uniqStrings(strs []string) []string { + // handle special case of a nil parameter + if strs == nil { + return nil + } + + // add each string to the slice as keys (used like a "set") + set := make(map[string]struct{}) + for _, s := range strs { + set[s] = struct{}{} + } + + // now copy all the keys over to a new slice and return it + result := []string{} + for key := range set { + result = append(result, key) + } + return result +} diff --git a/src/docrawler/util_test.go b/src/docrawler/util_test.go new file mode 100644 index 0000000..1ad3dca --- /dev/null +++ b/src/docrawler/util_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "sort" + "testing" +) + +func TestUniqStrings(t *testing.T) { + strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"} + uniq := uniqStrings(strs) + if len(uniq) != 5 { + t.Error("wrong number of strings after uniq'ing") + } + sort.Strings(uniq) + if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" { + t.Error("uniq values are in correct!") + } +} + +func TestUniqStringsNilParm(t *testing.T) { + uniq := uniqStrings(nil) + if uniq != nil { + t.Error("uniqStrings should return a nil slice when passed a nil slice") + } +} diff --git a/util/testfilter.sh b/util/testfilter.sh index 57cc952..e120961 100755 --- a/util/testfilter.sh +++ b/util/testfilter.sh @@ -2,6 +2,6 @@ # colorize the output of "go test" -sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/'' | \ +sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/'' | \ sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/'' | \ sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/'' From d3aeafeba57c5755cabd2d8b118539ff0528b6d6 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 16:28:36 -0800 Subject: [PATCH 03/22] update TODO --- docs/TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/TODO.md b/docs/TODO.md index 3e03d44..bcf97f6 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -6,6 +6,7 @@ * should only crawl these pages once * but should be included in the links part * when sorting links, "uniq" them as well +* be smart about index.html? ## Cleanup From 66f78b562fd350ec688e96cbffef5964d42a2479 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 16:34:09 -0800 Subject: [PATCH 04/22] flow: Created branch 'feature/refactor_state_management'. From 58614802b6cb2acd3712e1c1a57d1aa66e0240e6 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 18:14:12 -0800 Subject: [PATCH 05/22] renamed Page -> httpItem, and changed its members to unexported as well * tests all passing --- src/docrawler/docrawler.go | 44 ++++++++++++---------- src/docrawler/docrawler_test.go | 66 ++++++++++++++++----------------- src/docrawler/fetch.go | 16 ++++---- src/docrawler/fetch_test.go | 4 +- src/docrawler/output.go | 28 +++++++------- src/docrawler/page.go | 24 ++++++------ src/docrawler/page_test.go | 8 ++-- 7 files changed, 97 insertions(+), 93 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 48de706..70e8702 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -5,23 +5,27 @@ import ( "os" ) +type itemSlice []*httpItem +type itemMap map[string]*httpItem + + // docrawl begins crawling the site at "url" -func docrawl(url string) []*Page { +func docrawl(url string) itemSlice { // our result structure - var sitemap []*Page + var sitemap itemSlice // what we've already parsed, what we still need to parse - crawled := make(map[string]*Page) - queued := make(map[string]*Page) + crawled := make(itemMap) + queued := make(itemMap) // create first page and add to the queue - homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.URL.String()] = homepage + homepage, _ := newHTTPItem(nil, url) // TODO check error + queued[homepage.url.String()] = homepage // main loop for len(queued) > 0 { // pop a page off of queued - var dest *Page + var dest *httpItem var destURL string for destURL, dest = range queued { break @@ -29,52 +33,52 @@ func docrawl(url string) []*Page { delete(queued, destURL) // mark this page as complete - crawled[dest.URL.String()] = dest + crawled[dest.url.String()] = dest // add page to list sitemap = append(sitemap, dest) // make sure this URL has the same hostname as our first page - if dest.URL.Host != homepage.URL.Host { + if dest.url.Host != homepage.url.Host { // skip URLs associated with other Hosts - dest.Skipped = true + dest.skipped = true continue } // fetch page text, err := fetchPage(dest) if err != nil { - dest.Broken = true + dest.broken = true continue } // parse links title, links, err := parseLinks(text) if err != nil { - dest.Broken = true + dest.broken = true continue } - dest.Title = title + dest.title = title for _, l := range links { - p, err := NewPage(dest, l) + p, err := newHTTPItem(dest, l) if err != nil { continue // TODO check error } // see if we already know about this page - pCrawled, haveCrawled := crawled[p.URL.String()] - pQueued, haveQueued := queued[p.URL.String()] + pCrawled, haveCrawled := crawled[p.url.String()] + pQueued, haveQueued := queued[p.url.String()] if haveCrawled { // add the previously crawled page to children - dest.Children = append(dest.Children, pCrawled) + dest.children = append(dest.children, pCrawled) } else if haveQueued { // add the previously queued page to children - dest.Children = append(dest.Children, pQueued) + dest.children = append(dest.children, pQueued) } else { // add this new page (never seen) to children - dest.Children = append(dest.Children, p) + dest.children = append(dest.children, p) // .. and queue it up for crawling - queued[p.URL.String()] = p + queued[p.url.String()] = p } } } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6233cac..abd294f 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - page, err := NewPage(nil, baseURL+"fetch_test.html") + page, err := newHTTPItem(nil, baseURL+"fetch_test.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } body, err := fetchPage(page) if err != nil { @@ -82,19 +82,19 @@ func TestSimpleMap(t *testing.T) { // cycle through every page, counting stuff, finding specific pages // so that we can then verify certain aspects of each page skipped, children := 0, 0 - var homepage, aboutpage, skippage *Page + var homepage, aboutpage, skippage *httpItem for _, p := range pages { - if p.Title == "Home" { + if p.title == "Home" { homepage = p - } else if p.Title == "About Test" { + } else if p.title == "About Test" { aboutpage = p - } else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" { + } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.Skipped { + if p.skipped { skipped++ } - children += len(p.Children) + children += len(p.children) } // check page counts @@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) { } // check first page URL - if homepage.URL.String() != baseURL { + if homepage.url.String() != baseURL { t.Error("page URL is invalid") } // specific page number of children - if len(homepage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(homepage.Children), 4) + if len(homepage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } - if len(aboutpage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4) + if len(aboutpage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4) t.Fatal("got wrong number of links") } // make sure remote pages were skipped - if homepage.Skipped != false { - t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String()) + if homepage.skipped != false { + t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.Skipped != true { - t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String()) + if skippage.skipped != true { + t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } // verify page content // TODO find a better way to do this - //if homepage.Children[0].URL.String() != baseURL+"about.html" { - //t.Logf(" Got: %q\n", homepage.Children[0].URL.String()) + //if homepage.children[0].url.String() != baseURL+"about.html" { + //t.Logf(" Got: %q\n", homepage.children[0].url.String()) //t.Logf("Wanted: %q\n", baseURL+"about.html") //t.Error("link name is incorrect") //} - //if homepage.Children[1].URL.String() != baseURL+"assets/image.png" { - //t.Logf(" Got: %q\n", homepage.Children[1].URL.String()) + //if homepage.children[1].url.String() != baseURL+"assets/image.png" { + //t.Logf(" Got: %q\n", homepage.children[1].url.String()) //t.Logf("Wanted: %q\n", baseURL+"assets/image.png") //t.Error("link name is incorrect") //} - //if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" { - //t.Logf(" Got: %q\n", homepage.Children[3].URL.String()) + //if homepage.children[3].url.String() != baseURL+"scripts/blah.js" { + //t.Logf(" Got: %q\n", homepage.children[3].url.String()) //t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") //t.Error("link name is incorrect") //} @@ -157,28 +157,28 @@ func TestSimpleMap(t *testing.T) { // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - basepage, err := NewPage(nil, baseURL) + basepage, err := newHTTPItem(nil, baseURL) if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" { t.Error("problem fetching filetype") } - page, err := NewPage(basepage, "about.html") + page, err := newHTTPItem(basepage, "about.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" { t.Error("problem fetching filetype") } - page, err = NewPage(basepage, "assets/image.png") + page, err = newHTTPItem(basepage, "assets/image.png") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { - t.Logf("got %q, wanted %q", page.MediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" { + t.Logf("got %q, wanted %q", page.mediaType, "image/png") t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 9d41dc9..c7c83d9 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -13,9 +13,9 @@ var ( ) // fetchFiletype performs an http HEAD to get the media type, and sets it -// directly in Page.MediaType -func fetchFiletype(page *Page) error { - resp, err := http.Head(page.URL.String()) +// directly in httpItem.mediaType +func fetchFiletype(page *httpItem) error { + resp, err := http.Head(page.url.String()) if err != nil { return err } @@ -39,12 +39,12 @@ func fetchFiletype(page *Page) error { } // set and return - page.MediaType = mediatype + page.mediaType = mediatype return nil } -// fetchPage takes a Page, GETs it, and returns the body as a string -func fetchPage(page *Page) (string, error) { +// fetchPage takes a httpItem, GETs it, and returns the body as a string +func fetchPage(page *httpItem) (string, error) { // figure out the file type err := fetchFiletype(page) if err != nil { @@ -52,12 +52,12 @@ func fetchPage(page *Page) (string, error) { } // we only want to fetch html - if page.MediaType != "text/html" { + if page.mediaType != "text/html" { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.URL.String()) + resp, err := http.Get(page.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index 7cdff84..0cf42d1 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -6,7 +6,7 @@ import ( // TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedHeaderFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } @@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) { // TestFailedFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 87f5ad1..5b26247 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,32 +15,32 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*Page) []*Location { +func sitemapToLocations(pages []*httpItem) []*Location { // build a map of our pages - pageMap := make(map[string]*Page) + pageMap := make(map[string]*httpItem) for _, p := range pages { - pageMap[p.URL.String()] = p + pageMap[p.url.String()] = p } // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.MediaType == "text/html" { + if p.mediaType == "text/html" { // create a location for this page - l := &Location{URL: p.URL.String(), Title: p.Title} + l := &Location{URL: p.url.String(), Title: p.title} // add its children - for _, c := range p.Children { + for _, c := range p.children { // look up this child's media type from the root list of pages - //mediaType := pageMap[c.URL.String()].MediaType - if c.Skipped { - l.Remote = append(l.Remote, c.URL.String()) - } else if c.MediaType == "text/html" { - l.Links = append(l.Links, c.URL.String()) - } else if c.Broken { - l.Broken = append(l.Broken, c.URL.String()) + //mediaType := pageMap[c.url.String()].mediaType + if c.skipped { + l.Remote = append(l.Remote, c.url.String()) + } else if c.mediaType == "text/html" { + l.Links = append(l.Links, c.url.String()) + } else if c.broken { + l.Broken = append(l.Broken, c.url.String()) } else { - l.Assets = append(l.Assets, c.URL.String()) + l.Assets = append(l.Assets, c.url.String()) } } diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 9e8be32..d6f8ac8 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -4,22 +4,22 @@ import ( "net/url" ) -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page +// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type httpItem struct { + url *url.URL + title string + mediaType string + skipped bool + broken bool + children []*httpItem } -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { +// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} +func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { // determine the base URL so we can resulve this into a full URL baseURL := "" if referrer != nil { - baseURL = referrer.URL.String() + baseURL = referrer.url.String() } // resolve URL into a full absolute URL (non-relative) @@ -29,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) { } // create struct and return - return &Page{URL: u, Skipped: false, Broken: false}, nil + return &httpItem{url: u, skipped: false, broken: false}, nil } diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go index 217f290..d9a2143 100644 --- a/src/docrawler/page_test.go +++ b/src/docrawler/page_test.go @@ -4,10 +4,10 @@ import ( "testing" ) -// TestBadNewPage tests NewPage to see if we are getting expected failures -func TestBadNewPage(t *testing.T) { - _, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") +// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures +func TestBadnewHTTPItem(t *testing.T) { + _, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") if err == nil { - t.Error("tried creating a NewPage with an invalid URL but didn't get an error") + t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error") } } From d7cc760e5b2e9d361e63f6f70f8b2c73ce598eab Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 18:17:01 -0800 Subject: [PATCH 06/22] rename page.go -> item.go --- src/docrawler/{page.go => item.go} | 0 src/docrawler/{page_test.go => item_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/docrawler/{page.go => item.go} (100%) rename src/docrawler/{page_test.go => item_test.go} (100%) diff --git a/src/docrawler/page.go b/src/docrawler/item.go similarity index 100% rename from src/docrawler/page.go rename to src/docrawler/item.go diff --git a/src/docrawler/page_test.go b/src/docrawler/item_test.go similarity index 100% rename from src/docrawler/page_test.go rename to src/docrawler/item_test.go From 72cac47961901c1d8cefe0efbc069e84c37b66e6 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:13:52 -0800 Subject: [PATCH 07/22] docrawl() completely refactored to use goroutine per URL * outputs status * crawlStatus() function created (split out of docrawl()) * changed from using bool flags to linkType in single httpItem enum, tests updated to match this * unified where we set httpItem.linkType (inside fetchType), appropriate errors and returns added * renamed a lot of things from 'page' to 'item' * include refurl in httpItem * []*Location result is now sorted --- docs/TODO.md | 2 - src/docrawler/docrawler.go | 158 +++++++++++++++++++------------- src/docrawler/docrawler_test.go | 14 +-- src/docrawler/fetch.go | 34 +++++-- src/docrawler/item.go | 36 ++++++-- src/docrawler/output.go | 21 ++++- 6 files changed, 171 insertions(+), 94 deletions(-) diff --git a/docs/TODO.md b/docs/TODO.md index bcf97f6..52dea6e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -10,14 +10,12 @@ ## Cleanup -* rename Page -> crawlerState * helper function to do the whole crawl process * document code better * test with godoc ## Features -* some sort of status during crawl? * command line options, flag parsing * check RFCs on URL character sets, compliance, etc. * packagize parts of app diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 70e8702..af538a3 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,88 +2,120 @@ package main import ( "fmt" + "log" "os" + "time" ) -type itemSlice []*httpItem -type itemMap map[string]*httpItem +// docrawl begins crawling the site at "homeurl" +func docrawl(homeurl string) itemSlice { + // set of what we have already crawled + crawled := make(map[string]struct{}) + // map (of urlString -> httpItem) of our results + results := make(itemMap) -// docrawl begins crawling the site at "url" -func docrawl(url string) itemSlice { - // our result structure - var sitemap itemSlice + // a channel to receive our results on + rchan := make(chan *httpItem) - // what we've already parsed, what we still need to parse - crawled := make(itemMap) - queued := make(itemMap) + // start a ticker which we'll use to output status and check for completion + ticker := time.Tick(250 * time.Millisecond) - // create first page and add to the queue - homepage, _ := newHTTPItem(nil, url) // TODO check error - queued[homepage.url.String()] = homepage + // create first page's httpItem + homeitem, err := newHTTPItem(nil, homeurl) + if err != nil { + log.Fatal("unable to create httpItem for homeurl") + } - // main loop - for len(queued) > 0 { - // pop a page off of queued - var dest *httpItem - var destURL string - for destURL, dest = range queued { - break - } - delete(queued, destURL) + // set our number of outstanding pages (initially 1 to account for first page) + crawlingCount := 1 - // mark this page as complete - crawled[dest.url.String()] = dest + // start the home page crawl + crawled[homeitem.url.String()] = struct{}{} + go crawlItem(homeitem, rchan) - // add page to list - sitemap = append(sitemap, dest) + // wait for results + for { + select { + case r := <-rchan: // new results? + // add result to our results map + results[r.url.String()] = r - // make sure this URL has the same hostname as our first page - if dest.url.Host != homepage.url.Host { - // skip URLs associated with other Hosts - dest.skipped = true - continue - } + // decrease the outstanding page count by 1 + crawlingCount-- - // fetch page - text, err := fetchPage(dest) - if err != nil { - dest.broken = true - continue - } + // start crawly any new child pages we haven't yet crawled + for i, c := range r.children { + // see if we already have a result for this page, if so, point to that result + if existing, ok := results[c.url.String()]; ok { + r.children[i] = existing + continue + } - // parse links - title, links, err := parseLinks(text) - if err != nil { - dest.broken = true - continue - } - dest.title = title - for _, l := range links { - p, err := newHTTPItem(dest, l) - if err != nil { - continue // TODO check error + // see if we're already crawling this page (but maybe don't have results yet) + if _, ok := crawled[c.url.String()]; !ok { + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = struct{}{} + go crawlItem(c, rchan) + } } - // see if we already know about this page - pCrawled, haveCrawled := crawled[p.url.String()] - pQueued, haveQueued := queued[p.url.String()] - if haveCrawled { - // add the previously crawled page to children - dest.children = append(dest.children, pCrawled) - } else if haveQueued { - // add the previously queued page to children - dest.children = append(dest.children, pQueued) - } else { - // add this new page (never seen) to children - dest.children = append(dest.children, p) - // .. and queue it up for crawling - queued[p.url.String()] = p + case <-ticker: // our regular ticker. for status output and checking for completion. + // output status to console + log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount) + + // see if we're finished + if crawlingCount == 0 { + // finished! convert results map to a slice and return it + rslice := itemSlice{} + for _, v := range results { + rslice = append(rslice, v) + } + return rslice } } } +} - return sitemap +// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, +// and filling out its structure as much as possible +func crawlItem(item *httpItem, rchan chan<- *httpItem) { + // make sure this item is the same domain (i.e. URL "host part") as its referrer + if item.refurl != nil && item.url.Host != item.refurl.Host { + // skip URLs associated with other Hosts + item.linkType = tRemote + rchan <- item + return + } + + // fetch page + text, err := fetchPage(item) + if err != nil { + rchan <- item + return + } + + // parse links + title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed + if err != nil { + item.linkType = tBroken + rchan <- item + return + } + item.title = title + + // walk links and add them as children to the current item + for _, l := range links { + newItem, err := newHTTPItem(item, l) + if err != nil { + continue // TODO bad item + } + item.children = append(item.children, newItem) + } + + // send back our item struct now that it's all filled out + rchan <- item } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index abd294f..6be6209 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -91,7 +91,7 @@ func TestSimpleMap(t *testing.T) { } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.skipped { + if p.linkType == tRemote { skipped++ } children += len(p.children) @@ -127,11 +127,11 @@ func TestSimpleMap(t *testing.T) { } // make sure remote pages were skipped - if homepage.skipped != false { + if homepage.linkType == tRemote { t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.skipped != true { + if skippage.linkType != tRemote { t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } @@ -161,7 +161,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -169,7 +169,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -177,8 +177,8 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" { - t.Logf("got %q, wanted %q", page.mediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.linkType != tAsset { + t.Logf("got %q, wanted %q", page.linkType, tAsset) t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index c7c83d9..9cd67a6 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -9,19 +9,22 @@ import ( var ( errContentTypeNotFound = errors.New("no Content-Type header found") - errFetchError = errors.New("couldn't fetch page") + errFetchError = errors.New("couldn't fetch item") + errFileTypeUnknown = errors.New("couldn't determine file type") ) // fetchFiletype performs an http HEAD to get the media type, and sets it // directly in httpItem.mediaType -func fetchFiletype(page *httpItem) error { - resp, err := http.Head(page.url.String()) +func fetchFiletype(item *httpItem) error { + resp, err := http.Head(item.url.String()) if err != nil { + item.linkType = tBroken return err } // check response code if resp.StatusCode != http.StatusOK { + item.linkType = tBroken return errFetchError } @@ -29,35 +32,46 @@ func fetchFiletype(page *httpItem) error { contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found + item.linkType = tBroken return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) if err != nil { + item.linkType = tBroken return err } - // set and return - page.mediaType = mediatype + // success, set item type and return + if mediatype == "text/html" { + item.linkType = tHTMLPage + } else { + item.linkType = tAsset + } return nil } // fetchPage takes a httpItem, GETs it, and returns the body as a string -func fetchPage(page *httpItem) (string, error) { +func fetchPage(item *httpItem) (string, error) { // figure out the file type - err := fetchFiletype(page) + err := fetchFiletype(item) if err != nil { return "", err } - // we only want to fetch html - if page.mediaType != "text/html" { + // make sure we got a file type! + if item.linkType == tUnknown { + return "", errFileTypeUnknown + } + + // we only want to fetch html, so return if it's anything else + if item.linkType != tHTMLPage { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.url.String()) + resp, err := http.Get(item.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/item.go b/src/docrawler/item.go index d6f8ac8..cb9d90c 100644 --- a/src/docrawler/item.go +++ b/src/docrawler/item.go @@ -4,14 +4,30 @@ import ( "net/url" ) +// itemSlice is a convenience type for a slice of items +type itemSlice []*httpItem + +// itemMap convenience type for map of string's (URL) to items +type itemMap map[string]*httpItem + +// itemType is an enum so we know how to classify each item +type itemType int + +const ( + tUnknown itemType = iota + tHTMLPage + tAsset + tRemote + tBroken +) + // httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. type httpItem struct { - url *url.URL - title string - mediaType string - skipped bool - broken bool - children []*httpItem + url *url.URL + refurl *url.URL + title string + linkType itemType + children itemSlice } // newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} @@ -28,6 +44,12 @@ func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { return nil, err } + // figure out referrer url + var rurl *url.URL + if referrer != nil { + rurl = referrer.url + } + // create struct and return - return &httpItem{url: u, skipped: false, broken: false}, nil + return &httpItem{url: u, refurl: rurl}, nil } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 5b26247..01012de 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,7 +15,16 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*httpItem) []*Location { +// implement Location slice sorting (by URL) +type byURL []*Location + +// implement Sort interface on byURL (which is a []*Location) +func (l byURL) Len() int { return len(l) } +func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL } +func (l byURL) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling +func sitemapToLocations(pages itemSlice) []*Location { // build a map of our pages pageMap := make(map[string]*httpItem) for _, p := range pages { @@ -25,7 +34,7 @@ func sitemapToLocations(pages []*httpItem) []*Location { // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.mediaType == "text/html" { + if p.linkType == tHTMLPage { // create a location for this page l := &Location{URL: p.url.String(), Title: p.title} @@ -33,11 +42,11 @@ func sitemapToLocations(pages []*httpItem) []*Location { for _, c := range p.children { // look up this child's media type from the root list of pages //mediaType := pageMap[c.url.String()].mediaType - if c.skipped { + if c.linkType == tRemote { l.Remote = append(l.Remote, c.url.String()) - } else if c.mediaType == "text/html" { + } else if c.linkType == tHTMLPage { l.Links = append(l.Links, c.url.String()) - } else if c.broken { + } else if c.linkType == tBroken { l.Broken = append(l.Broken, c.url.String()) } else { l.Assets = append(l.Assets, c.url.String()) @@ -55,6 +64,8 @@ func sitemapToLocations(pages []*httpItem) []*Location { } } + // sort the Locations themselves and return + sort.Sort(byURL(locations)) return locations } From e3c730bdfcdb32bb5464f8f00dbb9d8eba228273 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:14:23 -0800 Subject: [PATCH 08/22] flow: Closed 'refactor_state_management'. From 7e18005473754df3940896379a48421e99087b88 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:24:59 -0800 Subject: [PATCH 09/22] remove page.go (merge miss from /develop) --- src/docrawler/page.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 src/docrawler/page.go diff --git a/src/docrawler/page.go b/src/docrawler/page.go deleted file mode 100644 index e0dcfcb..0000000 --- a/src/docrawler/page.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "net/url" -) - -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Identifier string - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page -} - -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { - // determine the base URL so we can resulve this into a full URL - baseURL := "" - if referrer != nil { - baseURL = referrer.URL.String() - } - - // resolve URL into a full absolute URL (non-relative) - u, err := resolveURL(baseURL, rawurl) - if err != nil { - return nil, err - } - - // create struct and return - p := &Page{URL: u, Skipped: false, Broken: false, Identifier: stripAnchorFromURL(u)} - return p, nil -} From 2c2cfc126c220c2c1fe19e0011627632fe0be38a Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:47:56 -0800 Subject: [PATCH 10/22] links are now uniq'd before marshaling * crawled is now a map to httpItem's (instead of struct{}) so we can connect pages mid-crawl * added duplicate link to html site, test updated to match * now any httpItem with linkType == tUnknown is listed as "Broken" in the output --- src/docrawler/docrawler.go | 21 +++++++++++++-------- src/docrawler/docrawler_test.go | 4 ++-- src/docrawler/output.go | 6 +++++- src/docrawler/testsite/index.html | 1 + 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index af538a3..3a72d18 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -10,7 +10,7 @@ import ( // docrawl begins crawling the site at "homeurl" func docrawl(homeurl string) itemSlice { // set of what we have already crawled - crawled := make(map[string]struct{}) + crawled := make(itemMap) // map (of urlString -> httpItem) of our results results := make(itemMap) @@ -31,7 +31,7 @@ func docrawl(homeurl string) itemSlice { crawlingCount := 1 // start the home page crawl - crawled[homeitem.url.String()] = struct{}{} + crawled[homeitem.url.String()] = homeitem go crawlItem(homeitem, rchan) // wait for results @@ -46,19 +46,24 @@ func docrawl(homeurl string) itemSlice { // start crawly any new child pages we haven't yet crawled for i, c := range r.children { - // see if we already have a result for this page, if so, point to that result + // see if we already have a result for this page + // if so, point to that result if existing, ok := results[c.url.String()]; ok { r.children[i] = existing continue } // see if we're already crawling this page (but maybe don't have results yet) - if _, ok := crawled[c.url.String()]; !ok { - // haven't crawled this one yet, do so now - crawlingCount++ - crawled[c.url.String()] = struct{}{} - go crawlItem(c, rchan) + // if so, point to that item (we will have it as a result later!) + if existing, ok := crawled[c.url.String()]; ok { + r.children[i] = existing + continue } + + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = c + go crawlItem(c, rchan) } case <-ticker: // our regular ticker. for status output and checking for completion. diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6be6209..c908ba0 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -102,7 +102,7 @@ func TestSimpleMap(t *testing.T) { t.Logf("got %v, wanted %v\n", len(pages), 6) t.Fatal("got wrong number of pages") } - if children != 8 { + if children != 9 { t.Logf("got %v, wanted %v\n", children, 8) t.Fatal("got wrong number of total children") } @@ -117,7 +117,7 @@ func TestSimpleMap(t *testing.T) { } // specific page number of children - if len(homepage.children) != 4 { + if len(homepage.children) != 5 { t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 426599b..58361bf 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -48,9 +48,13 @@ func sitemapToLocations(pages itemSlice) []*Location { l.Links = append(l.Links, c.url.String()) } else if c.linkType == tBroken { l.Broken = append(l.Broken, c.url.String()) - } else { + } else if c.linkType == tAsset { l.Assets = append(l.Assets, c.url.String()) + } else { + // unknown link here, which means it failed to crawl, let's call it "broken" + l.Broken = append(l.Broken, c.url.String()) } + } // now uniq & sort the children slices diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html index ce8b225..98b1179 100644 --- a/src/docrawler/testsite/index.html +++ b/src/docrawler/testsite/index.html @@ -5,6 +5,7 @@ fooey! + duplicate!