flow: Closed <feature> 'use_crawler_workers'.

src/docrawler/docrawler.go

@@ -7,9 +7,9 @@ import (
 	"time"
 )
 
-// docrawl begins crawling the site at "homeurl"
+// doCrawl begins crawling the site at "homeurl"
 func doCrawl(homeurl string) itemSlice {
-	// set of what we have already crawled
+	// set of what we have already crawled, our results
 	crawled := make(itemMap)
 
 	// a set of crawled URLs which have been cleaned
@@ -17,11 +17,14 @@ func doCrawl(homeurl string) itemSlice {
 	// already crawled http://a.com/index.html, or vice versa
 	crawledStripped := make(itemMap)
 
-	// map (of urlString -> httpItem) of our results
-	results := make(itemMap)
+	// channels to send work to and receive our results on
+	rxchan := make(chan *httpItem)
+	txchan := make(chan *httpItem)
 
-	// a channel to receive our results on
-	rchan := make(chan *httpItem)
+	// spin up our crawler workers
+	for i := 0; i < 10; i++ {
+		go crawlListner(txchan, rxchan)
+	}
 
 	// start a ticker which we'll use to output status and check for completion
 	ticker := time.Tick(250 * time.Millisecond)
@@ -29,7 +32,7 @@ func doCrawl(homeurl string) itemSlice {
 	// create first page's httpItem
 	homeitem, err := newHTTPItem(nil, homeurl)
 	if err != nil {
-		log.Fatal("unable to create httpItem for homeurl")
+		return nil
 	}
 
 	// set our number of outstanding pages (initially 1 to account for first page)
@@ -38,14 +41,14 @@ func doCrawl(homeurl string) itemSlice {
 	// start the home page crawl
 	crawled[homeitem.url.String()] = homeitem
 	crawledStripped[stripURL(homeitem.url)] = homeitem
-	go crawlItem(homeitem, rchan)
+	txchan <- homeitem
 
 	// wait for results
 	for {
 		select {
-		case r := <-rchan: // new results?
+		case r := <-rxchan: // new results?
 			// add result to our results map
-			results[r.url.String()] = r
+			crawled[r.url.String()] = r
 
 			// decrease the outstanding page count by 1
 			crawlingCount--
@@ -54,7 +57,7 @@ func doCrawl(homeurl string) itemSlice {
 			for i, c := range r.children {
 				// see if we already have a result for this page
 				// if so, point to that result
-				if existing, ok := results[c.url.String()]; ok {
+				if existing, ok := crawled[c.url.String()]; ok {
 					r.children[i] = existing
 					continue
 				}
@@ -80,18 +83,24 @@ func doCrawl(homeurl string) itemSlice {
 				crawlingCount++
 				crawled[c.url.String()] = c
 				crawledStripped[stripURL(c.url)] = c
-				go crawlItem(c, rchan)
+				go func(newItem *httpItem) {
+					txchan <- newItem
+				}(c)
 			}
 
 		case <-ticker: // our regular ticker. for status output and checking for completion.
 			// output status to console
-			log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount)
+			log.Printf("Crawled %v links, have %v left.\n", len(crawled), crawlingCount)
 
 			// see if we're finished
 			if crawlingCount == 0 {
+				// close our channels, signalling any workers to exit
+				close(txchan)
+				close(rxchan)
+
 				// finished! convert results map to a slice and return it
 				rslice := itemSlice{}
-				for _, v := range results {
+				for _, v := range crawled {
 					rslice = append(rslice, v)
 				}
 				return rslice
@@ -100,31 +109,40 @@ func doCrawl(homeurl string) itemSlice {
 	}
 }
 
+// crawlListner is a for+select wrapper around crawlItem that listens
+// for new jobs and sends them off to crawlItem
+func crawlListner(txchan <-chan *httpItem, rxchan chan<- *httpItem) {
+	for {
+		select {
+		case newJob, ok := <-txchan:
+			if !ok {
+				// channel closed, finish
+				return
+			}
+			// perform the crawl
+			rxchan <- crawlItem(newJob)
+		}
+	}
+}
+
 // crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
 // and filling out its structure as much as possible
-func crawlItem(item *httpItem, rchan chan<- *httpItem) {
+func crawlItem(item *httpItem) *httpItem {
 	// make sure this item is the same domain (i.e. URL "host part") as its referrer
 	if item.refurl != nil && item.url.Host != item.refurl.Host {
 		// skip URLs associated with other Hosts
 		item.linkType = tRemote
-		rchan <- item
-		return
+		return item
 	}
 
 	// fetch page
 	text, err := fetchPage(item)
 	if err != nil {
-		rchan <- item
-		return
+		return item
 	}
 
 	// parse links
-	title, links := parseLinks(text) // TODO remove error from parseLinks, not needed
-	if err != nil {
-		item.linkType = tBroken
-		rchan <- item
-		return
-	}
+	title, links := parseLinks(text)
 	item.title = title
 
 	// walk links and add them as children to the current item
@@ -137,13 +155,22 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) {
 	}
 
 	// send back our item struct now that it's all filled out
-	rchan <- item
+	return item
 }
 
 func main() {
+	// see if we've got no arguments
+	if len(os.Args) == 1 {
+		fmt.Printf("\nD.O. Crawler 1.0  Copyright (c) 2015 Stephen Waits <steve@waits.net>  2015-02-17\n\n")
+		fmt.Printf("usage: %v <URLs...>\n\n", os.Args[0])
+		os.Exit(1)
+	}
 	// crawl each URL on the command line
 	for _, u := range os.Args[1:] {
 		pages := doCrawl(u)
+		if pages == nil {
+			log.Fatalf("unable to crawl %q, may be an invalid URL\n", u)
+		}
 		l := sitemapToLocations(pages)
 		j, _ := locationsToJSON(l)
 		fmt.Println(j)