docrawl() completely refactored to use goroutine per URL

* outputs status
* crawlStatus() function created (split out of docrawl())
* changed from using bool flags to linkType in single httpItem enum, tests updated to match this
* unified where we set httpItem.linkType (inside fetchType), appropriate errors and returns added
* renamed a lot of things from 'page' to 'item'
* include refurl in httpItem
* []*Location result is now sorted

docs/TODO.md

@@ -10,14 +10,12 @@
 
 ## Cleanup
 
-* rename Page -> crawlerState
 * helper function to do the whole crawl process
 * document code better
   * test with godoc
 
 ## Features
 
-* some sort of status during crawl?
 * command line options, flag parsing
 * check RFCs on URL character sets, compliance, etc.
 * packagize parts of app

src/docrawler/docrawler.go

@@ -2,88 +2,120 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
+	"time"
 )
 
-type itemSlice []*httpItem
+// docrawl begins crawling the site at "homeurl"
-type itemMap   map[string]*httpItem
+func docrawl(homeurl string) itemSlice {
+	// set of what we have already crawled
+	crawled := make(map[string]struct{})
 
+	// map (of urlString -> httpItem) of our results
+	results := make(itemMap)
 
-// docrawl begins crawling the site at "url"
+	// a channel to receive our results on
-func docrawl(url string) itemSlice {
+	rchan := make(chan *httpItem)
-	// our result structure
-	var sitemap itemSlice
 
-	// what we've already parsed, what we still need to parse
+	// start a ticker which we'll use to output status and check for completion
-	crawled := make(itemMap)
+	ticker := time.Tick(250 * time.Millisecond)
-	queued := make(itemMap)
 
-	// create first page and add to the queue
+	// create first page's httpItem
-	homepage, _ := newHTTPItem(nil, url) // TODO check error
+	homeitem, err := newHTTPItem(nil, homeurl)
-	queued[homepage.url.String()] = homepage
+	if err != nil {
+		log.Fatal("unable to create httpItem for homeurl")
+	}
 
-	// main loop
+	// set our number of outstanding pages (initially 1 to account for first page)
-	for len(queued) > 0 {
+	crawlingCount := 1
-		// pop a page off of queued
-		var dest *httpItem
-		var destURL string
-		for destURL, dest = range queued {
-			break
-		}
-		delete(queued, destURL)
 
-		// mark this page as complete
+	// start the home page crawl
-		crawled[dest.url.String()] = dest
+	crawled[homeitem.url.String()] = struct{}{}
+	go crawlItem(homeitem, rchan)
 
-		// add page to list
+	// wait for results
-		sitemap = append(sitemap, dest)
+	for {
+		select {
+		case r := <-rchan: // new results?
+			// add result to our results map
+			results[r.url.String()] = r
 
-		// make sure this URL has the same hostname as our first page
+			// decrease the outstanding page count by 1
-		if dest.url.Host != homepage.url.Host {
+			crawlingCount--
-			// skip URLs associated with other Hosts
-			dest.skipped = true
-			continue
-		}
 
-		// fetch page
+			// start crawly any new child pages we haven't yet crawled
-		text, err := fetchPage(dest)
+			for i, c := range r.children {
-		if err != nil {
+				// see if we already have a result for this page, if so, point to that result
-			dest.broken = true
+				if existing, ok := results[c.url.String()]; ok {
-			continue
+					r.children[i] = existing
-		}
+					continue
+				}
 
-		// parse links
+				// see if we're already crawling this page (but maybe don't have results yet)
-		title, links, err := parseLinks(text)
+				if _, ok := crawled[c.url.String()]; !ok {
-		if err != nil {
+					// haven't crawled this one yet, do so now
-			dest.broken = true
+					crawlingCount++
-			continue
+					crawled[c.url.String()] = struct{}{}
-		}
+					go crawlItem(c, rchan)
-		dest.title = title
+				}
-		for _, l := range links {
-			p, err := newHTTPItem(dest, l)
-			if err != nil {
-				continue // TODO check error
 			}
 
-			// see if we already know about this page
+		case <-ticker: // our regular ticker. for status output and checking for completion.
-			pCrawled, haveCrawled := crawled[p.url.String()]
+			// output status to console
-			pQueued, haveQueued := queued[p.url.String()]
+			log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount)
-			if haveCrawled {
+
-				// add the previously crawled page to children
+			// see if we're finished
-				dest.children = append(dest.children, pCrawled)
+			if crawlingCount == 0 {
-			} else if haveQueued {
+				// finished! convert results map to a slice and return it
-				// add the previously queued page to children
+				rslice := itemSlice{}
-				dest.children = append(dest.children, pQueued)
+				for _, v := range results {
-			} else {
+					rslice = append(rslice, v)
-				// add this new page (never seen) to children
+				}
-				dest.children = append(dest.children, p)
+				return rslice
-				// .. and queue it up for crawling
-				queued[p.url.String()] = p
 			}
 		}
 	}
+}
 
-	return sitemap
+// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
+// and filling out its structure as much as possible
+func crawlItem(item *httpItem, rchan chan<- *httpItem) {
+	// make sure this item is the same domain (i.e. URL "host part") as its referrer
+	if item.refurl != nil && item.url.Host != item.refurl.Host {
+		// skip URLs associated with other Hosts
+		item.linkType = tRemote
+		rchan <- item
+		return
+	}
+
+	// fetch page
+	text, err := fetchPage(item)
+	if err != nil {
+		rchan <- item
+		return
+	}
+
+	// parse links
+	title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed
+	if err != nil {
+		item.linkType = tBroken
+		rchan <- item
+		return
+	}
+	item.title = title
+
+	// walk links and add them as children to the current item
+	for _, l := range links {
+		newItem, err := newHTTPItem(item, l)
+		if err != nil {
+			continue // TODO bad item
+		}
+		item.children = append(item.children, newItem)
+	}
+
+	// send back our item struct now that it's all filled out
+	rchan <- item
 }
 
 func main() {

src/docrawler/docrawler_test.go

@@ -91,7 +91,7 @@ func TestSimpleMap(t *testing.T) {
 		} else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" {
 			skippage = p
 		}
-		if p.skipped {
+		if p.linkType == tRemote {
 			skipped++
 		}
 		children += len(p.children)
@@ -127,11 +127,11 @@ func TestSimpleMap(t *testing.T) {
 	}
 
 	// make sure remote pages were skipped
-	if homepage.skipped != false {
+	if homepage.linkType == tRemote {
 		t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String())
 		t.Error("child was skipped")
 	}
-	if skippage.skipped != true {
+	if skippage.linkType != tRemote {
 		t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String())
 		t.Error("child wasn't skipped")
 	}
@@ -161,7 +161,7 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" {
+	if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage {
 		t.Error("problem fetching filetype")
 	}
 
@@ -169,7 +169,7 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" {
+	if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage {
 		t.Error("problem fetching filetype")
 	}
 
@@ -177,8 +177,8 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" {
+	if err := fetchFiletype(page); err != nil || page.linkType != tAsset {
-		t.Logf("got %q, wanted %q", page.mediaType, "image/png")
+		t.Logf("got %q, wanted %q", page.linkType, tAsset)
 		t.Error("problem fetching filetype")
 	}
 }

src/docrawler/fetch.go

@@ -9,19 +9,22 @@ import (
 
 var (
 	errContentTypeNotFound = errors.New("no Content-Type header found")
-	errFetchError          = errors.New("couldn't fetch page")
+	errFetchError          = errors.New("couldn't fetch item")
+	errFileTypeUnknown     = errors.New("couldn't determine file type")
 )
 
 // fetchFiletype performs an http HEAD to get the media type, and sets it
 // directly in httpItem.mediaType
-func fetchFiletype(page *httpItem) error {
+func fetchFiletype(item *httpItem) error {
-	resp, err := http.Head(page.url.String())
+	resp, err := http.Head(item.url.String())
 	if err != nil {
+		item.linkType = tBroken
 		return err
 	}
 
 	// check response code
 	if resp.StatusCode != http.StatusOK {
+		item.linkType = tBroken
 		return errFetchError
 	}
 
@@ -29,35 +32,46 @@ func fetchFiletype(page *httpItem) error {
 	contentType, ok := resp.Header["Content-Type"]
 	if !ok {
 		// no Content-Type header found
+		item.linkType = tBroken
 		return errContentTypeNotFound
 	}
 
 	// parse mime type
 	mediatype, _, err := mime.ParseMediaType(contentType[0])
 	if err != nil {
+		item.linkType = tBroken
 		return err
 	}
 
-	// set and return
+	// success, set item type and return
-	page.mediaType = mediatype
+	if mediatype == "text/html" {
+		item.linkType = tHTMLPage
+	} else {
+		item.linkType = tAsset
+	}
 	return nil
 }
 
 // fetchPage takes a httpItem, GETs it, and returns the body as a string
-func fetchPage(page *httpItem) (string, error) {
+func fetchPage(item *httpItem) (string, error) {
 	// figure out the file type
-	err := fetchFiletype(page)
+	err := fetchFiletype(item)
 	if err != nil {
 		return "", err
 	}
 
-	// we only want to fetch html
+	// make sure we got a file type!
-	if page.mediaType != "text/html" {
+	if item.linkType == tUnknown {
+		return "", errFileTypeUnknown
+	}
+
+	// we only want to fetch html, so return if it's anything else
+	if item.linkType != tHTMLPage {
 		return "", nil // but this isn't an error!
 	}
 
 	// GET the url
-	resp, err := http.Get(page.url.String())
+	resp, err := http.Get(item.url.String())
 	if err != nil {
 		return "", err
 	}

src/docrawler/item.go

@@ -4,14 +4,30 @@ import (
 	"net/url"
 )
 
+// itemSlice is a convenience type for a slice of items
+type itemSlice []*httpItem
+
+// itemMap convenience type for map of string's (URL) to items
+type itemMap map[string]*httpItem
+
+// itemType is an enum so we know how to classify each item
+type itemType int
+
+const (
+	tUnknown itemType = iota
+	tHTMLPage
+	tAsset
+	tRemote
+	tBroken
+)
+
 // httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc.
 type httpItem struct {
-	url       *url.URL
+	url      *url.URL
-	title     string
+	refurl   *url.URL
-	mediaType string
+	title    string
-	skipped   bool
+	linkType itemType
-	broken    bool
+	children itemSlice
-	children  []*httpItem
 }
 
 // newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{}
@@ -28,6 +44,12 @@ func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) {
 		return nil, err
 	}
 
+	// figure out referrer url
+	var rurl *url.URL
+	if referrer != nil {
+		rurl = referrer.url
+	}
+
 	// create struct and return
-	return &httpItem{url: u, skipped: false, broken: false}, nil
+	return &httpItem{url: u, refurl: rurl}, nil
 }

src/docrawler/output.go

@@ -15,7 +15,16 @@ type Location struct {
 	Remote []string
 }
 
-func sitemapToLocations(pages []*httpItem) []*Location {
+// implement Location slice sorting (by URL)
+type byURL []*Location
+
+// implement Sort interface on byURL (which is a []*Location)
+func (l byURL) Len() int           { return len(l) }
+func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL }
+func (l byURL) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
+
+// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling
+func sitemapToLocations(pages itemSlice) []*Location {
 	// build a map of our pages
 	pageMap := make(map[string]*httpItem)
 	for _, p := range pages {
@@ -25,7 +34,7 @@ func sitemapToLocations(pages []*httpItem) []*Location {
 	// build a slice of locations (one per page)
 	var locations []*Location
 	for _, p := range pages {
-		if p.mediaType == "text/html" {
+		if p.linkType == tHTMLPage {
 			// create a location for this page
 			l := &Location{URL: p.url.String(), Title: p.title}
 
@@ -33,11 +42,11 @@ func sitemapToLocations(pages []*httpItem) []*Location {
 			for _, c := range p.children {
 				// look up this child's media type from the root list of pages
 				//mediaType := pageMap[c.url.String()].mediaType
-				if c.skipped {
+				if c.linkType == tRemote {
 					l.Remote = append(l.Remote, c.url.String())
-				} else if c.mediaType == "text/html" {
+				} else if c.linkType == tHTMLPage {
 					l.Links = append(l.Links, c.url.String())
-				} else if c.broken {
+				} else if c.linkType == tBroken {
 					l.Broken = append(l.Broken, c.url.String())
 				} else {
 					l.Assets = append(l.Assets, c.url.String())
@@ -55,6 +64,8 @@ func sitemapToLocations(pages []*httpItem) []*Location {
 		}
 	}
 
+	// sort the Locations themselves and return
+	sort.Sort(byURL(locations))
 	return locations
 }