flow: Merged <feature> 'remove_error_from_parselinks' to <develop> ('develop').

docs/TODO.md

@@ -5,7 +5,6 @@
 * deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.)
   * should only crawl these pages once
   * but should be included in the links part
-* when sorting links, "uniq" them as well
 * be smart about index.html?
 
 ## Cleanup

src/docrawler/docrawler.go

@@ -102,7 +102,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) {
 	}
 
 	// parse links
-	title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed
+	title, links := parseLinks(text) // TODO remove error from parseLinks, not needed
 	if err != nil {
 		item.linkType = tBroken
 		rchan <- item

src/docrawler/parse.go

@@ -11,7 +11,7 @@ var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
 // parseLinks takes a string and attempts to parse any html title and all links out of it,
 // and returns a slice of the captures found
-func parseLinks(s string) (string, []string, error) {
+func parseLinks(s string) (string, []string) {
 	// find the title, or default to empty
 	title := ""
 	titleMatches := reTitle.FindStringSubmatch(s)
@@ -25,5 +25,5 @@ func parseLinks(s string) (string, []string, error) {
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	return title, results, nil
+	return title, results
 }

src/docrawler/parse_test.go

@@ -16,10 +16,7 @@ func TestSimpleParse(t *testing.T) {
 		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	title, matches, err := parseLinks(doc)
-	if err != nil {
-		t.Error(err)
-	}
+	title, matches := parseLinks(doc)
 	if title != "Test Page" {
 		t.Error("got wrong title")
 	}