links are now uniq'd before marshaling

* crawled is now a map to httpItem's (instead of struct{}) so we can connect pages mid-crawl
* added duplicate link to html site, test updated to match
* now any httpItem with linkType == tUnknown is listed as "Broken" in the output

src/docrawler/docrawler.go

@@ -10,7 +10,7 @@ import (
 // docrawl begins crawling the site at "homeurl"
 func docrawl(homeurl string) itemSlice {
 	// set of what we have already crawled
-	crawled := make(map[string]struct{})
+	crawled := make(itemMap)
 
 	// map (of urlString -> httpItem) of our results
 	results := make(itemMap)
@@ -31,7 +31,7 @@ func docrawl(homeurl string) itemSlice {
 	crawlingCount := 1
 
 	// start the home page crawl
-	crawled[homeitem.url.String()] = struct{}{}
+	crawled[homeitem.url.String()] = homeitem
 	go crawlItem(homeitem, rchan)
 
 	// wait for results
@@ -46,19 +46,24 @@ func docrawl(homeurl string) itemSlice {
 
 			// start crawly any new child pages we haven't yet crawled
 			for i, c := range r.children {
-				// see if we already have a result for this page, if so, point to that result
+				// see if we already have a result for this page
+				// if so, point to that result
 				if existing, ok := results[c.url.String()]; ok {
 					r.children[i] = existing
 					continue
 				}
 
 				// see if we're already crawling this page (but maybe don't have results yet)
-				if _, ok := crawled[c.url.String()]; !ok {
-					// haven't crawled this one yet, do so now
-					crawlingCount++
-					crawled[c.url.String()] = struct{}{}
-					go crawlItem(c, rchan)
+				// if so, point to that item (we will have it as a result later!)
+				if existing, ok := crawled[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
 				}
+
+				// haven't crawled this one yet, do so now
+				crawlingCount++
+				crawled[c.url.String()] = c
+				go crawlItem(c, rchan)
 			}
 
 		case <-ticker: // our regular ticker. for status output and checking for completion.

src/docrawler/docrawler_test.go

@@ -102,7 +102,7 @@ func TestSimpleMap(t *testing.T) {
 		t.Logf("got %v, wanted %v\n", len(pages), 6)
 		t.Fatal("got wrong number of pages")
 	}
-	if children != 8 {
+	if children != 9 {
 		t.Logf("got %v, wanted %v\n", children, 8)
 		t.Fatal("got wrong number of total children")
 	}
@@ -117,7 +117,7 @@ func TestSimpleMap(t *testing.T) {
 	}
 
 	// specific page number of children
-	if len(homepage.children) != 4 {
+	if len(homepage.children) != 5 {
 		t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
 		t.Fatal("got wrong number of links")
 	}

src/docrawler/output.go

@@ -48,9 +48,13 @@ func sitemapToLocations(pages itemSlice) []*Location {
 					l.Links = append(l.Links, c.url.String())
 				} else if c.linkType == tBroken {
 					l.Broken = append(l.Broken, c.url.String())
-				} else {
+				} else if c.linkType == tAsset {
 					l.Assets = append(l.Assets, c.url.String())
+				} else {
+					// unknown link here, which means it failed to crawl, let's call it "broken"
+					l.Broken = append(l.Broken, c.url.String())
 				}
+
 			}
 
 			// now uniq & sort the children slices

src/docrawler/testsite/index.html

@@ -5,6 +5,7 @@
 	<body>
 		<img src="/assets/image.png"/>
 		<a href="/about.html">fooey!</a>
+		<a href="/about.html">duplicate!</a>
 		<script src="scripts/blah.js"/>
 		<a href="/zzzbroken.html">oops!</a>
 	</body>