From d9009cdbaac75ff9c1aaff0df54e164fe7830bf1 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 15:50:30 -0800 Subject: [PATCH 1/5] flow: Created branch 'feature/unique_links_from_each_page'. From 2e997084171e4e8abfc2b11b576752088bbacea2 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 16:26:16 -0800 Subject: [PATCH 2/5] won't recrawl pages that are the same after stripping #anchor (blah.html#about, blah.html#contact, etc.) * but broken * committing on feature branch and will come back to this --- docs/TODO.md | 1 + src/docrawler/docrawler.go | 10 +++++----- src/docrawler/output.go | 6 +++++- src/docrawler/page.go | 16 +++++++++------- src/docrawler/url.go | 9 +++++++++ src/docrawler/url_test.go | 11 +++++++++++ src/docrawler/util.go | 23 +++++++++++++++++++++++ src/docrawler/util_test.go | 25 +++++++++++++++++++++++++ util/testfilter.sh | 2 +- 9 files changed, 89 insertions(+), 14 deletions(-) create mode 100644 src/docrawler/util.go create mode 100644 src/docrawler/util_test.go diff --git a/docs/TODO.md b/docs/TODO.md index 3e03d44..450c69e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -13,6 +13,7 @@ * helper function to do the whole crawl process * document code better * test with godoc +* omit fields in json when nil? (Broken, etc.) ## Features diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 48de706..08ac36f 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -16,7 +16,7 @@ func docrawl(url string) []*Page { // create first page and add to the queue homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.URL.String()] = homepage + queued[homepage.Identifier] = homepage // main loop for len(queued) > 0 { @@ -29,7 +29,7 @@ func docrawl(url string) []*Page { delete(queued, destURL) // mark this page as complete - crawled[dest.URL.String()] = dest + crawled[dest.Identifier] = dest // add page to list sitemap = append(sitemap, dest) @@ -62,8 +62,8 @@ func docrawl(url string) []*Page { } // see if we already know about this page - pCrawled, haveCrawled := crawled[p.URL.String()] - pQueued, haveQueued := queued[p.URL.String()] + pCrawled, haveCrawled := crawled[p.Identifier] + pQueued, haveQueued := queued[p.Identifier] if haveCrawled { // add the previously crawled page to children dest.Children = append(dest.Children, pCrawled) @@ -74,7 +74,7 @@ func docrawl(url string) []*Page { // add this new page (never seen) to children dest.Children = append(dest.Children, p) // .. and queue it up for crawling - queued[p.URL.String()] = p + queued[p.Identifier] = p } } } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 87f5ad1..2b6b959 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -44,7 +44,11 @@ func sitemapToLocations(pages []*Page) []*Location { } } - // now sort the children slices + // now uniq & sort the children slices + l.Remote = uniqStrings(l.Remote) + l.Links = uniqStrings(l.Links) + l.Broken = uniqStrings(l.Broken) + l.Assets = uniqStrings(l.Assets) sort.Strings(l.Remote) sort.Strings(l.Links) sort.Strings(l.Broken) diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 9e8be32..e0dcfcb 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -6,12 +6,13 @@ import ( // Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { - URL *url.URL - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page + URL *url.URL + Identifier string + Title string + MediaType string + Skipped bool + Broken bool + Children []*Page } // NewPage takes a referring Page + a URL and returns a new &Page{} @@ -29,5 +30,6 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) { } // create struct and return - return &Page{URL: u, Skipped: false, Broken: false}, nil + p := &Page{URL: u, Skipped: false, Broken: false, Identifier: stripAnchorFromURL(u)} + return p, nil } diff --git a/src/docrawler/url.go b/src/docrawler/url.go index d66bc44..d98657a 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -49,3 +49,12 @@ func resolveURL(referringURL string, currentURL string) (*url.URL, error) { // return the URL we ended up with, and the error from checkURL return uResolved, checkURL(uResolved) } + +func stripAnchorFromURL(u *url.URL) string { + // copy the url locally + ucopy := *u + + // blank out the anchor, return URI string + ucopy.Fragment = "" + return ucopy.String() +} diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index b0867e8..07a4a9b 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -68,3 +68,14 @@ func TestBadResolve(t* testing.T) { } } */ + +func TestURLStripAnchor(t *testing.T) { + u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo") + if err != nil { + t.Fatal("couldn't parse a URL") + } + stripped := stripAnchorFromURL(u) + if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" { + t.Error("stripping anchor #foo from URL failed") + } +} diff --git a/src/docrawler/util.go b/src/docrawler/util.go new file mode 100644 index 0000000..d28531b --- /dev/null +++ b/src/docrawler/util.go @@ -0,0 +1,23 @@ +package main + +// uniqStrings takes a slice of strings and removes any duplicates +// note: does not guarantee any order (or stability of order) +func uniqStrings(strs []string) []string { + // handle special case of a nil parameter + if strs == nil { + return nil + } + + // add each string to the slice as keys (used like a "set") + set := make(map[string]struct{}) + for _, s := range strs { + set[s] = struct{}{} + } + + // now copy all the keys over to a new slice and return it + result := []string{} + for key := range set { + result = append(result, key) + } + return result +} diff --git a/src/docrawler/util_test.go b/src/docrawler/util_test.go new file mode 100644 index 0000000..1ad3dca --- /dev/null +++ b/src/docrawler/util_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "sort" + "testing" +) + +func TestUniqStrings(t *testing.T) { + strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"} + uniq := uniqStrings(strs) + if len(uniq) != 5 { + t.Error("wrong number of strings after uniq'ing") + } + sort.Strings(uniq) + if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" { + t.Error("uniq values are in correct!") + } +} + +func TestUniqStringsNilParm(t *testing.T) { + uniq := uniqStrings(nil) + if uniq != nil { + t.Error("uniqStrings should return a nil slice when passed a nil slice") + } +} diff --git a/util/testfilter.sh b/util/testfilter.sh index 57cc952..e120961 100755 --- a/util/testfilter.sh +++ b/util/testfilter.sh @@ -2,6 +2,6 @@ # colorize the output of "go test" -sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/'' | \ +sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/'' | \ sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/'' | \ sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/'' From 7e18005473754df3940896379a48421e99087b88 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:24:59 -0800 Subject: [PATCH 3/5] remove page.go (merge miss from /develop) --- src/docrawler/page.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 src/docrawler/page.go diff --git a/src/docrawler/page.go b/src/docrawler/page.go deleted file mode 100644 index e0dcfcb..0000000 --- a/src/docrawler/page.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "net/url" -) - -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Identifier string - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page -} - -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { - // determine the base URL so we can resulve this into a full URL - baseURL := "" - if referrer != nil { - baseURL = referrer.URL.String() - } - - // resolve URL into a full absolute URL (non-relative) - u, err := resolveURL(baseURL, rawurl) - if err != nil { - return nil, err - } - - // create struct and return - p := &Page{URL: u, Skipped: false, Broken: false, Identifier: stripAnchorFromURL(u)} - return p, nil -} From 2c2cfc126c220c2c1fe19e0011627632fe0be38a Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Mon, 16 Feb 2015 22:47:56 -0800 Subject: [PATCH 4/5] links are now uniq'd before marshaling * crawled is now a map to httpItem's (instead of struct{}) so we can connect pages mid-crawl * added duplicate link to html site, test updated to match * now any httpItem with linkType == tUnknown is listed as "Broken" in the output --- src/docrawler/docrawler.go | 21 +++++++++++++-------- src/docrawler/docrawler_test.go | 4 ++-- src/docrawler/output.go | 6 +++++- src/docrawler/testsite/index.html | 1 + 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index af538a3..3a72d18 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -10,7 +10,7 @@ import ( // docrawl begins crawling the site at "homeurl" func docrawl(homeurl string) itemSlice { // set of what we have already crawled - crawled := make(map[string]struct{}) + crawled := make(itemMap) // map (of urlString -> httpItem) of our results results := make(itemMap) @@ -31,7 +31,7 @@ func docrawl(homeurl string) itemSlice { crawlingCount := 1 // start the home page crawl - crawled[homeitem.url.String()] = struct{}{} + crawled[homeitem.url.String()] = homeitem go crawlItem(homeitem, rchan) // wait for results @@ -46,19 +46,24 @@ func docrawl(homeurl string) itemSlice { // start crawly any new child pages we haven't yet crawled for i, c := range r.children { - // see if we already have a result for this page, if so, point to that result + // see if we already have a result for this page + // if so, point to that result if existing, ok := results[c.url.String()]; ok { r.children[i] = existing continue } // see if we're already crawling this page (but maybe don't have results yet) - if _, ok := crawled[c.url.String()]; !ok { - // haven't crawled this one yet, do so now - crawlingCount++ - crawled[c.url.String()] = struct{}{} - go crawlItem(c, rchan) + // if so, point to that item (we will have it as a result later!) + if existing, ok := crawled[c.url.String()]; ok { + r.children[i] = existing + continue } + + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = c + go crawlItem(c, rchan) } case <-ticker: // our regular ticker. for status output and checking for completion. diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6be6209..c908ba0 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -102,7 +102,7 @@ func TestSimpleMap(t *testing.T) { t.Logf("got %v, wanted %v\n", len(pages), 6) t.Fatal("got wrong number of pages") } - if children != 8 { + if children != 9 { t.Logf("got %v, wanted %v\n", children, 8) t.Fatal("got wrong number of total children") } @@ -117,7 +117,7 @@ func TestSimpleMap(t *testing.T) { } // specific page number of children - if len(homepage.children) != 4 { + if len(homepage.children) != 5 { t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 426599b..58361bf 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -48,9 +48,13 @@ func sitemapToLocations(pages itemSlice) []*Location { l.Links = append(l.Links, c.url.String()) } else if c.linkType == tBroken { l.Broken = append(l.Broken, c.url.String()) - } else { + } else if c.linkType == tAsset { l.Assets = append(l.Assets, c.url.String()) + } else { + // unknown link here, which means it failed to crawl, let's call it "broken" + l.Broken = append(l.Broken, c.url.String()) } + } // now uniq & sort the children slices diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html index ce8b225..98b1179 100644 --- a/src/docrawler/testsite/index.html +++ b/src/docrawler/testsite/index.html @@ -5,6 +5,7 @@ fooey! + duplicate!