Merge from develop

src/docrawler/docrawler.go

@@ -116,7 +116,7 @@ func crawlWorker(txchan <-chan *httpItem, rxchan chan<- *httpItem) {
 	// read off the incoming item channel forever
 	for newJob := range txchan {
 		// perform the crawl
-		crawlItem(newJob)
+		newJob.crawlItem()
 		// return the result
 		rxchan <- newJob
 	}

src/docrawler/docrawler_test.go

@@ -62,7 +62,7 @@ func TestSimpleFetch(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	body, err := fetchItem(page)
+	body, err := page.fetchItem()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -161,7 +161,7 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage {
+	if err := basepage.fetchFiletype(); err != nil || basepage.linkType != tHTMLPage {
 		t.Error("problem fetching filetype")
 	}
 
@@ -169,7 +169,7 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage {
+	if err := page.fetchFiletype(); err != nil || page.linkType != tHTMLPage {
 		t.Error("problem fetching filetype")
 	}
 
@@ -177,7 +177,7 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.linkType != tAsset {
+	if err := page.fetchFiletype(); err != nil || page.linkType != tAsset {
 		t.Logf("got %q, wanted %q", page.linkType, tAsset)
 		t.Error("problem fetching filetype")
 	}

src/docrawler/fetch.go

@@ -16,7 +16,7 @@ var (
 
 // fetchFiletype performs an http HEAD to get the media type, and sets it
 // directly in httpItem.mediaType
-func fetchFiletype(item *httpItem) error {
+func (item *httpItem) fetchFiletype() error {
 	resp, err := http.Head(item.url.String())
 	if err != nil {
 		item.linkType = tBroken
@@ -54,9 +54,9 @@ func fetchFiletype(item *httpItem) error {
 }
 
 // fetchPage takes a httpItem, GETs it, and returns the body as a string
-func fetchItem(item *httpItem) (string, error) {
+func (item *httpItem) fetchItem() (string, error) {
 	// figure out the file type
-	err := fetchFiletype(item)
+	err := item.fetchFiletype()
 	if err != nil {
 		return "", err
 	}
@@ -95,7 +95,7 @@ func fetchItem(item *httpItem) (string, error) {
 
 // crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
 // and filling out its structure as much as possible
-func crawlItem(item *httpItem) {
+func (item *httpItem) crawlItem() {
 	// make sure this item is the same domain (i.e. URL "host part") as its referrer
 	if item.refurl != nil && item.url.Host != item.refurl.Host {
 		// skip URLs associated with other Hosts
@@ -104,7 +104,7 @@ func crawlItem(item *httpItem) {
 	}
 
 	// fetch page
-	text, err := fetchItem(item)
+	text, err := item.fetchItem()
 	if err != nil {
 		return
 	}

src/docrawler/fetch_test.go

@@ -10,7 +10,7 @@ func TestFailedHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}
-	if err := fetchFiletype(page); err == nil {
+	if err := page.fetchFiletype(); err == nil {
 		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
 	}
 }
@@ -21,7 +21,7 @@ func TestFailedFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}
-	if _, err := fetchItem(page); err == nil {
+	if _, err := page.fetchItem(); err == nil {
 		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
 	}
 }