diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 780f84b..ff0b233 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -116,7 +116,7 @@ func crawlWorker(txchan <-chan *httpItem, rxchan chan<- *httpItem) { // read off the incoming item channel forever for newJob := range txchan { // perform the crawl - crawlItem(newJob) + newJob.crawlItem() // return the result rxchan <- newJob } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 5c11cc9..ab50248 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -62,7 +62,7 @@ func TestSimpleFetch(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - body, err := fetchItem(page) + body, err := page.fetchItem() if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage { + if err := basepage.fetchFiletype(); err != nil || basepage.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -169,7 +169,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage { + if err := page.fetchFiletype(); err != nil || page.linkType != tHTMLPage { t.Error("problem fetching filetype") } @@ -177,7 +177,7 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.linkType != tAsset { + if err := page.fetchFiletype(); err != nil || page.linkType != tAsset { t.Logf("got %q, wanted %q", page.linkType, tAsset) t.Error("problem fetching filetype") } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index a24c954..ac5fcca 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -16,7 +16,7 @@ var ( // fetchFiletype performs an http HEAD to get the media type, and sets it // directly in httpItem.mediaType -func fetchFiletype(item *httpItem) error { +func (item *httpItem) fetchFiletype() error { resp, err := http.Head(item.url.String()) if err != nil { item.linkType = tBroken @@ -54,9 +54,9 @@ func fetchFiletype(item *httpItem) error { } // fetchPage takes a httpItem, GETs it, and returns the body as a string -func fetchItem(item *httpItem) (string, error) { +func (item *httpItem) fetchItem() (string, error) { // figure out the file type - err := fetchFiletype(item) + err := item.fetchFiletype() if err != nil { return "", err } @@ -95,7 +95,7 @@ func fetchItem(item *httpItem) (string, error) { // crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, // and filling out its structure as much as possible -func crawlItem(item *httpItem) { +func (item *httpItem) crawlItem() { // make sure this item is the same domain (i.e. URL "host part") as its referrer if item.refurl != nil && item.url.Host != item.refurl.Host { // skip URLs associated with other Hosts @@ -104,7 +104,7 @@ func crawlItem(item *httpItem) { } // fetch page - text, err := fetchItem(item) + text, err := item.fetchItem() if err != nil { return } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index c666a87..a68b0aa 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -10,7 +10,7 @@ func TestFailedHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New Page struct") } - if err := fetchFiletype(page); err == nil { + if err := page.fetchFiletype(); err == nil { t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype") } } @@ -21,7 +21,7 @@ func TestFailedFetching(t *testing.T) { if err != nil { t.Error("problem creating New Page struct") } - if _, err := fetchItem(page); err == nil { + if _, err := page.fetchItem(); err == nil { t.Error("tired fetching bogus page but didn't get nil back from fetchPage") } }