From 13b84ee452c3d3aa753dd8bcc81749f7628e81bc Mon Sep 17 00:00:00 2001
From: Stephen Waits <steve@waits.net>
Date: Tue, 17 Feb 2015 09:28:24 -0800
Subject: [PATCH] completion of crawler worker mode. tests still passing!

---
 src/docrawler/docrawler.go | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go
index c2f094d..26a84f8 100644
--- a/src/docrawler/docrawler.go
+++ b/src/docrawler/docrawler.go
@@ -119,36 +119,30 @@ func crawlListner(txchan <-chan *httpItem, rxchan chan<- *httpItem) {
 				// channel closed, finish
 				return
 			}
-			crawlItem(newJob, rxchan)
+			// perform the crawl
+			rxchan <- crawlItem(newJob)
 		}
 	}
 }
 
 // crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
 // and filling out its structure as much as possible
-func crawlItem(item *httpItem, rxchan chan<- *httpItem) {
+func crawlItem(item *httpItem) *httpItem {
 	// make sure this item is the same domain (i.e. URL "host part") as its referrer
 	if item.refurl != nil && item.url.Host != item.refurl.Host {
 		// skip URLs associated with other Hosts
 		item.linkType = tRemote
-		rxchan <- item
-		return
+		return item
 	}
 
 	// fetch page
 	text, err := fetchPage(item)
 	if err != nil {
-		rxchan <- item
-		return
+		return item
 	}
 
 	// parse links
-	title, links := parseLinks(text) // TODO remove error from parseLinks, not needed
-	if err != nil {
-		item.linkType = tBroken
-		rxchan <- item
-		return
-	}
+	title, links := parseLinks(text)
 	item.title = title
 
 	// walk links and add them as children to the current item
@@ -161,7 +155,7 @@ func crawlItem(item *httpItem, rxchan chan<- *httpItem) {
 	}
 
 	// send back our item struct now that it's all filled out
-	rxchan <- item
+	return item
 }
 
 func main() {