won't recrawl pages that are the same after stripping #anchor (blah.html#about, blah.html#contact, etc.)

* but broken
* committing on feature branch and will come back to this

docs/TODO.md

@@ -13,6 +13,7 @@
 * helper function to do the whole crawl process
 * document code better
   * test with godoc
+* omit fields in json when nil? (Broken, etc.)
 
 ## Features
 

src/docrawler/docrawler.go

@@ -16,7 +16,7 @@ func docrawl(url string) []*Page {
 
 	// create first page and add to the queue
 	homepage, _ := NewPage(nil, url) // TODO check error
-	queued[homepage.URL.String()] = homepage
+	queued[homepage.Identifier] = homepage
 
 	// main loop
 	for len(queued) > 0 {
@@ -29,7 +29,7 @@ func docrawl(url string) []*Page {
 		delete(queued, destURL)
 
 		// mark this page as complete
-		crawled[dest.URL.String()] = dest
+		crawled[dest.Identifier] = dest
 
 		// add page to list
 		sitemap = append(sitemap, dest)
@@ -62,8 +62,8 @@ func docrawl(url string) []*Page {
 			}
 
 			// see if we already know about this page
-			pCrawled, haveCrawled := crawled[p.URL.String()]
-			pQueued, haveQueued := queued[p.URL.String()]
+			pCrawled, haveCrawled := crawled[p.Identifier]
+			pQueued, haveQueued := queued[p.Identifier]
 			if haveCrawled {
 				// add the previously crawled page to children
 				dest.Children = append(dest.Children, pCrawled)
@@ -74,7 +74,7 @@ func docrawl(url string) []*Page {
 				// add this new page (never seen) to children
 				dest.Children = append(dest.Children, p)
 				// .. and queue it up for crawling
-				queued[p.URL.String()] = p
+				queued[p.Identifier] = p
 			}
 		}
 	}

src/docrawler/output.go

@@ -44,7 +44,11 @@ func sitemapToLocations(pages []*Page) []*Location {
 				}
 			}
 
-			// now sort the children slices
+			// now uniq & sort the children slices
+			l.Remote = uniqStrings(l.Remote)
+			l.Links = uniqStrings(l.Links)
+			l.Broken = uniqStrings(l.Broken)
+			l.Assets = uniqStrings(l.Assets)
 			sort.Strings(l.Remote)
 			sort.Strings(l.Links)
 			sort.Strings(l.Broken)

src/docrawler/page.go

@@ -7,6 +7,7 @@ import (
 // Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
 type Page struct {
 	URL        *url.URL
+	Identifier string
 	Title      string
 	MediaType  string
 	Skipped    bool
@@ -29,5 +30,6 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) {
 	}
 
 	// create struct and return
-	return &Page{URL: u, Skipped: false, Broken: false}, nil
+	p := &Page{URL: u, Skipped: false, Broken: false, Identifier: stripAnchorFromURL(u)}
+	return p, nil
 }

src/docrawler/url.go

@@ -49,3 +49,12 @@ func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
 	// return the URL we ended up with, and the error from checkURL
 	return uResolved, checkURL(uResolved)
 }
+
+func stripAnchorFromURL(u *url.URL) string {
+	// copy the url locally
+	ucopy := *u
+
+	// blank out the anchor, return URI string
+	ucopy.Fragment = ""
+	return ucopy.String()
+}

src/docrawler/url_test.go

@@ -68,3 +68,14 @@ func TestBadResolve(t* testing.T) {
 	}
 }
 */
+
+func TestURLStripAnchor(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Fatal("couldn't parse a URL")
+	}
+	stripped := stripAnchorFromURL(u)
+	if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" {
+		t.Error("stripping anchor #foo from URL failed")
+	}
+}

src/docrawler/util.go

@@ -0,0 +1,23 @@
+package main
+
+// uniqStrings takes a slice of strings and removes any duplicates
+// note: does not guarantee any order (or stability of order)
+func uniqStrings(strs []string) []string {
+	// handle special case of a nil parameter
+	if strs == nil {
+		return nil
+	}
+
+	// add each string to the slice as keys (used like a "set")
+	set := make(map[string]struct{})
+	for _, s := range strs {
+		set[s] = struct{}{}
+	}
+
+	// now copy all the keys over to a new slice and return it
+	result := []string{}
+	for key := range set {
+		result = append(result, key)
+	}
+	return result
+}

src/docrawler/util_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestUniqStrings(t *testing.T) {
+	strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"}
+	uniq := uniqStrings(strs)
+	if len(uniq) != 5 {
+		t.Error("wrong number of strings after uniq'ing")
+	}
+	sort.Strings(uniq)
+	if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" {
+		t.Error("uniq values are in correct!")
+	}
+}
+
+func TestUniqStringsNilParm(t *testing.T) {
+	uniq := uniqStrings(nil)
+	if uniq != nil {
+		t.Error("uniqStrings should return a nil slice when passed a nil slice")
+	}
+}

util/testfilter.sh

@@ -2,6 +2,6 @@
 
 # colorize the output of "go test"
 
-sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
+sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
 	sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/''   | \
 	sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/''