diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 90f55bd..dca03c1 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -102,12 +102,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) { } // parse links - title, links := parseLinks(text) // TODO remove error from parseLinks, not needed - if err != nil { - item.linkType = tBroken - rchan <- item - return - } + title, links := parseLinks(text) item.title = title // walk links and add them as children to the current item diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index bab34f0..187f504 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -74,11 +74,13 @@ func fetchPage(item *httpItem) (string, error) { // GET the url resp, err := http.Get(item.url.String()) if err != nil { + // NOTE http.Head was already successful, this is highly likely to succeed return "", err } // check response code if resp.StatusCode != http.StatusOK { + // NOTE we already got StatusOK for http.Head, this is highly likely to succeed return "", errFetchError }