diff --git a/docs/TODO.md b/docs/TODO.md index 5fe18f4..a6d55f4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -5,7 +5,6 @@ * deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.) * should only crawl these pages once * but should be included in the links part -* when sorting links, "uniq" them as well * be smart about index.html? ## Cleanup diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 2502944..90f55bd 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -102,7 +102,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) { } // parse links - title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed + title, links := parseLinks(text) // TODO remove error from parseLinks, not needed if err != nil { item.linkType = tBroken rchan <- item diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 1c735d6..4eb31b3 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -11,7 +11,7 @@ var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) // parseLinks takes a string and attempts to parse any html title and all links out of it, // and returns a slice of the captures found -func parseLinks(s string) (string, []string, error) { +func parseLinks(s string) (string, []string) { // find the title, or default to empty title := "" titleMatches := reTitle.FindStringSubmatch(s) @@ -25,5 +25,5 @@ func parseLinks(s string) (string, []string, error) { for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } - return title, results, nil + return title, results } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index a04b869..ccc08a3 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -16,10 +16,7 @@ func TestSimpleParse(t *testing.T) {