flow: Closed <feature> 'fix_anchor_duplication'.

* so now http://a.com/index.html and http://a.com/ are understood equivalents
* as are http://a.com/x.html#blog and http://a.com/x.html (etc.)
* this meant a little more bookkeeping in docrawler, but not horrible
* stripAnchorFromURL because stripURL because it now handles anchor and path suffix stripping
* and url tests were written to support this

docs/TODO.md

@@ -2,17 +2,12 @@
 
 ## Bugs
 
-* deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.)
-  * should only crawl these pages once
-  * but should be included in the links part
-* when sorting links, "uniq" them as well
-* be smart about index.html?
-
 ## Cleanup
 
 * helper function to do the whole crawl process
 * document code better
   * test with godoc
+* omit fields in json when nil? (Broken, etc.)
 
 ## Features
 

src/docrawler/docrawler.go

@@ -8,9 +8,14 @@ import (
 )
 
 // docrawl begins crawling the site at "homeurl"
-func docrawl(homeurl string) itemSlice {
+func doCrawl(homeurl string) itemSlice {
 	// set of what we have already crawled
-	crawled := make(map[string]struct{})
+	crawled := make(itemMap)
+
+	// a set of crawled URLs which have been cleaned
+	// useful so that we don't crawl http://a.com/index.html#about if we've
+	// already crawled http://a.com/index.html, or vice versa
+	crawledStripped := make(itemMap)
 
 	// map (of urlString -> httpItem) of our results
 	results := make(itemMap)
@@ -31,7 +36,8 @@ func docrawl(homeurl string) itemSlice {
 	crawlingCount := 1
 
 	// start the home page crawl
-	crawled[homeitem.url.String()] = struct{}{}
+	crawled[homeitem.url.String()] = homeitem
+	crawledStripped[stripURL(homeitem.url)] = homeitem
 	go crawlItem(homeitem, rchan)
 
 	// wait for results
@@ -46,19 +52,35 @@ func docrawl(homeurl string) itemSlice {
 
 			// start crawly any new child pages we haven't yet crawled
 			for i, c := range r.children {
-				// see if we already have a result for this page, if so, point to that result
+				// see if we already have a result for this page
+				// if so, point to that result
 				if existing, ok := results[c.url.String()]; ok {
 					r.children[i] = existing
 					continue
 				}
 
 				// see if we're already crawling this page (but maybe don't have results yet)
-				if _, ok := crawled[c.url.String()]; !ok {
-					// haven't crawled this one yet, do so now
-					crawlingCount++
-					crawled[c.url.String()] = struct{}{}
-					go crawlItem(c, rchan)
+				// if so, point to that item (we will have it as a result later!)
+				if existing, ok := crawled[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
 				}
+				if existing, ok := crawledStripped[stripURL(c.url)]; ok {
+					// we crawled a different version of this same page
+					// i.e. same page, different anchor. we don't need to crawl it again
+					// but we do need to keep it in our results, so copy the existing
+					// struct over for everything except the URLs
+					r.children[i].title = existing.title
+					r.children[i].linkType = existing.linkType
+					r.children[i].children = existing.children
+					continue
+				}
+
+				// haven't crawled this one yet, do so now
+				crawlingCount++
+				crawled[c.url.String()] = c
+				crawledStripped[stripURL(c.url)] = c
+				go crawlItem(c, rchan)
 			}
 
 		case <-ticker: // our regular ticker. for status output and checking for completion.
@@ -97,7 +119,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) {
 	}
 
 	// parse links
-	title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed
+	title, links := parseLinks(text) // TODO remove error from parseLinks, not needed
 	if err != nil {
 		item.linkType = tBroken
 		rchan <- item
@@ -121,7 +143,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) {
 func main() {
 	// crawl each URL on the command line
 	for _, u := range os.Args[1:] {
-		pages := docrawl(u)
+		pages := doCrawl(u)
 		l := sitemapToLocations(pages)
 		j, _ := locationsToJSON(l)
 		fmt.Println(j)

src/docrawler/docrawler_test.go

@@ -76,7 +76,7 @@ func TestSimpleFetch(t *testing.T) {
 
 // TestSimpleMap figures out the site map for the site in baseURL
 func TestSimpleMap(t *testing.T) {
-	pages := docrawl(baseURL)
+	pages := doCrawl(baseURL)
 
 	// because our crawl is non-deterministic, we have to do a complete
 	// cycle through every page, counting stuff, finding specific pages
@@ -102,7 +102,7 @@ func TestSimpleMap(t *testing.T) {
 		t.Logf("got %v, wanted %v\n", len(pages), 6)
 		t.Fatal("got wrong number of pages")
 	}
-	if children != 8 {
+	if children != 9 {
 		t.Logf("got %v, wanted %v\n", children, 8)
 		t.Fatal("got wrong number of total children")
 	}
@@ -117,7 +117,7 @@ func TestSimpleMap(t *testing.T) {
 	}
 
 	// specific page number of children
-	if len(homepage.children) != 4 {
+	if len(homepage.children) != 5 {
 		t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
 		t.Fatal("got wrong number of links")
 	}
@@ -185,7 +185,7 @@ func TestHeaderFetching(t *testing.T) {
 
 // TestJsonOutput gets a sitemap and then converts it to json
 func TestJsonOutput(t *testing.T) {
-	pages := docrawl(baseURL)
+	pages := doCrawl(baseURL)
 	l := sitemapToLocations(pages)
 	if len(l) != 2 {
 		t.Error("sitemapToLocations has the wrong number of locations")

src/docrawler/fetch.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 )
 
+// custom errors
 var (
 	errContentTypeNotFound = errors.New("no Content-Type header found")
 	errFetchError          = errors.New("couldn't fetch item")

src/docrawler/item.go

@@ -13,6 +13,7 @@ type itemMap map[string]*httpItem
 // itemType is an enum so we know how to classify each item
 type itemType int
 
+// enums for itemType
 const (
 	tUnknown itemType = iota
 	tHTMLPage

src/docrawler/output.go

@@ -48,12 +48,20 @@ func sitemapToLocations(pages itemSlice) []*Location {
 					l.Links = append(l.Links, c.url.String())
 				} else if c.linkType == tBroken {
 					l.Broken = append(l.Broken, c.url.String())
-				} else {
+				} else if c.linkType == tAsset {
 					l.Assets = append(l.Assets, c.url.String())
+				} else {
+					// unknown link here, which means it failed to crawl, let's call it "broken"
+					l.Broken = append(l.Broken, c.url.String())
 				}
+
 			}
 
-			// now sort the children slices
+			// now uniq & sort the children slices
+			l.Remote = uniqStrings(l.Remote)
+			l.Links = uniqStrings(l.Links)
+			l.Broken = uniqStrings(l.Broken)
+			l.Assets = uniqStrings(l.Assets)
 			sort.Strings(l.Remote)
 			sort.Strings(l.Links)
 			sort.Strings(l.Broken)
@@ -69,6 +77,7 @@ func sitemapToLocations(pages itemSlice) []*Location {
 	return locations
 }
 
+// locationsToJSON takes a *Location slice and marshals it into a JSON string
 func locationsToJSON(locations []*Location) (string, error) {
 	b, err := json.Marshal(locations)
 	if err != nil {

src/docrawler/parse.go

@@ -11,7 +11,7 @@ var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
 // parseLinks takes a string and attempts to parse any html title and all links out of it,
 // and returns a slice of the captures found
-func parseLinks(s string) (string, []string, error) {
+func parseLinks(s string) (string, []string) {
 	// find the title, or default to empty
 	title := ""
 	titleMatches := reTitle.FindStringSubmatch(s)
@@ -25,5 +25,5 @@ func parseLinks(s string) (string, []string, error) {
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	return title, results, nil
+	return title, results
 }

src/docrawler/parse_test.go

@@ -16,10 +16,7 @@ func TestSimpleParse(t *testing.T) {
 		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	title, matches, err := parseLinks(doc)
-	if err != nil {
-		t.Error(err)
-	}
+	title, matches := parseLinks(doc)
 	if title != "Test Page" {
 		t.Error("got wrong title")
 	}

src/docrawler/testsite/index.html

@@ -5,6 +5,7 @@
 	<body>
 		<img src="/assets/image.png"/>
 		<a href="/about.html">fooey!</a>
+		<a href="/about.html">duplicate!</a>
 		<script src="scripts/blah.js"/>
 		<a href="/zzzbroken.html">oops!</a>
 	</body>

src/docrawler/url.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 )
 
+// custom errors
 var (
 	errInvalidURL = errors.New("this URL can't be parsed successfully")
 )
@@ -49,3 +50,27 @@ func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
 	// return the URL we ended up with, and the error from checkURL
 	return uResolved, checkURL(uResolved)
 }
+
+// stripURL returns a version of the URL without the "Fragment" part,
+// which is anything after the '#' character, so:
+// http://a.com/blah.html#anchor becomes http://a.com/blah.html
+// ... and without files like "index.htm" in the Path (since that's
+// equivalent to "/"
+func stripURL(u *url.URL) string {
+	// copy the url locally
+	ucopy := *u
+
+	// blank out the anchor, return URI string
+	ucopy.Fragment = ""
+
+	// remove index.html if it exists
+	suffixes := []string{"index.html", "index.htm"}
+	for _, s := range suffixes {
+		if strings.HasSuffix(strings.ToLower(ucopy.Path), s) {
+			ucopy.Path = ucopy.Path[0 : len(ucopy.Path)-len(s)]
+		}
+	}
+
+	// return new stripped down URL string
+	return ucopy.String()
+}

src/docrawler/url_test.go

@@ -68,3 +68,37 @@ func TestBadResolve(t* testing.T) {
 	}
 }
 */
+
+// TestURLStripAnchor tests the stripURL() function
+func TestURLStripAnchor(t *testing.T) {
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/index.html?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/index.htm?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+}

src/docrawler/util.go

@@ -0,0 +1,23 @@
+package main
+
+// uniqStrings takes a slice of strings and removes any duplicates
+// note: does not guarantee any order (or stability of order)
+func uniqStrings(strs []string) []string {
+	// handle special case of a nil parameter
+	if strs == nil {
+		return nil
+	}
+
+	// add each string to the slice as keys (used like a "set")
+	set := make(map[string]struct{})
+	for _, s := range strs {
+		set[s] = struct{}{}
+	}
+
+	// now copy all the keys over to a new slice and return it
+	result := []string{}
+	for key := range set {
+		result = append(result, key)
+	}
+	return result
+}

src/docrawler/util_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"sort"
+	"testing"
+)
+
+// TestUniqStrings passes a slice of strings to uniqStrings() to see if it removes
+// all duplicates
+func TestUniqStrings(t *testing.T) {
+	strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"}
+	uniq := uniqStrings(strs)
+	if len(uniq) != 5 {
+		t.Error("wrong number of strings after uniq'ing")
+	}
+	sort.Strings(uniq)
+	if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" {
+		t.Error("uniq values are in correct!")
+	}
+}
+
+// TestUniqStringsNilParm passes a nil slice to uniqStrings to be sure it returns nil
+func TestUniqStringsNilParm(t *testing.T) {
+	uniq := uniqStrings(nil)
+	if uniq != nil {
+		t.Error("uniqStrings should return a nil slice when passed a nil slice")
+	}
+}

util/testfilter.sh

@@ -2,6 +2,6 @@
 
 # colorize the output of "go test"
 
-sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
+sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
 	sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/''   | \
 	sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/''