flow: Closed <feature> 'use_crawler_workers'.

* so now http://a.com/index.html and http://a.com/ are understood equivalents
* as are http://a.com/x.html#blog and http://a.com/x.html (etc.)
* this meant a little more bookkeeping in docrawler, but not horrible
* stripAnchorFromURL because stripURL because it now handles anchor and path suffix stripping
* and url tests were written to support this

.hgignore

@@ -1,2 +1,3 @@
 syntax: glob
-bin
+bin
+cover.out

Makefile

@@ -3,18 +3,20 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest run clean env vendor
+.PHONY: build fmt lint vet test autotest cover doc run clean env stats vendor
 
 # configuration
 APPNAME := docrawler
 VENDOR := .vendor
 PACKAGES :=                     \
+	golang.org/x/tools/cmd/cover  \
+	golang.org/x/tools/cmd/godoc  \
 	golang.org/x/tools/cmd/vet    \
 	github.com/golang/lint/golint
 
 # override GOPATH to be our vendor directory
-export GOPATH := ${PWD}/${VENDOR}
-export PATH := ${PWD}/${VENDOR}/bin:${PATH}
+export GOPATH := ${PWD}/${VENDOR}:${PWD}
+export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH}
 
 default: build
 
@@ -24,35 +26,54 @@ build: fmt lint vet
 fmt:
 	go fmt ./src/...
 
+# TODO golint is not accepting the "./src/..." for some reason (from within make)
 lint:
-	golint ./src
+	golint ./src/${APPNAME}/*.go
 
 vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./util/testfilter.sh
+	go test -v ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \
-	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+		xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+
+cover:
+	go test -coverprofile cover.out ./src/...  && go tool cover -html cover.out
+
+doc:
+	godoc -http=:6060 -index
 
 run: build
 	./bin/${APPNAME}
 
 clean:
-	rm -f ./bin/${APPNAME}
+	rm -f ./bin/${APPNAME} ./cover.out
 
 env:
 	go env
 
+stats:
+	@echo ===============================================================================
+	@echo Tests
+	@echo ===============================================================================
+	@cloc --match-f=_test.go ./src
+	@echo
+	@echo
+	@echo ===============================================================================
+	@echo Everything Else
+	@echo ===============================================================================
+	@cloc --not-match-f=_test.go ./src
+
 # vendor
 # * wipes our vendor directory completely
-# * fetches all packages in ${PACKAGES}
+# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)
 # * removes source control dirs so packages are vendorable (commitable)
 vendor:
-	rm -dRf ./${VENDOR}/src                          && \
-	go get -u ${PACKAGES}                            && \
+	rm -dRf ./${VENDOR}                              && \
+	GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES}    && \
 	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
 	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
 	rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \

docs/Design.md

@@ -2,38 +2,38 @@
 
 ## Minimum Features
 
-This is the assignment (directly) from DO.
+This is the assignment (directly) from **REDACTED**.
 
-* We do ask that you put a heavy emphasis on testing in the assignment.
-* written in a modern language
-* limited to one domain - so when crawling digitalocean.com it would crawl all pages within the digitalocean.com domain, but not follow the links to our Facebook or Instagram accounts or subdomains like cloud.digitalocean.com. 
-* It should be Given a URL
-* it should output a site map
-* it should show which static assets each page depends on, and the links between pages.
-* Choose the most appropriate data structure to store & display this site map.
-* Build this as you would build something for production - it's fine if you don't finish everything, but focus on code quality and write tests.
-* We're interested in how you code and how you test your code.
+✓ We do ask that you put a heavy emphasis on testing in the assignment.
+ written in a modern language
+✓ limited to one domain - so when crawling REDACTED.com it would crawl all pages within the REDACTED.com domain, but not follow the links to our Facebook or Instagram accounts or subdomains like cloud.REDACTED.com. 
+✓ It should be Given a URL
+✓ it should output a site map
+✓ it should show which static assets each page depends on, and the links between pages.
+✓ Choose the most appropriate data structure to store & display this site map.
+✓ Build this as you would build something for production - it's fine if you don't finish everything, but focus on code quality and write tests.
+✓ We're interested in how you code and how you test your code.
 
 ## Desired
 
 These are features I'd like to include.
 
 * efficient, optimal
-* handles base URL properly (for root & relative links)
-* documented
-* uses standard Go practices
-* uses Go stdlib (only!)
-* thoroughly tested
+✓ handles base URL properly (for root & relative links)
+✓ documented
+✓ uses standard Go practices
+✓ uses Go stdlib (only!)
+✓ thoroughly tested
 * configurable on command line (defaults are for company specs)
 * variable output formats: json, dot
-* adheres to URL RFC (as far as case sensitivity, acceptable character sets, etc.)
+✓ adheres to URL RFC (as far as case sensitivity, acceptable character sets, etc.)
 * go gettable
-* go fmt'ed, vet'ed, lint'ed
+✓ go fmt'ed, vet'ed, lint'ed
 * polite - robots.txt support
-* robust - detect infinite loops
+✓ robust - detect infinite loops
 * throttling
 * use Go context pattern
-* supports http & https
+✓ supports http & https
 
 ## Architecture
 
@@ -41,9 +41,8 @@ These are features I'd like to include.
 * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines)
 * pipeline may allows us to scale concurrency at the bottlenecks
 * could use Interfaces for things like Parser, Getter, but I think best avoided until needed
-* launch goroutine per fetch
-* when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it
-
+✓ launch goroutine per fetch
+✓ when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it
 
 ## Wishlist
 

docs/TODO.md

@@ -0,0 +1,16 @@
+# To Do
+
+## Bugs
+
+## Cleanup
+
+* helper function to do the whole crawl process
+* document code better
+  * test with godoc
+* omit fields in json when nil? (Broken, etc.)
+
+## Features
+
+* command line options, flag parsing
+* check RFCs on URL character sets, compliance, etc.
+* packagize parts of app

src/docrawler/docrawler.go

@@ -1,6 +1,178 @@
 package main
 
-import ()
+import (
+	"fmt"
+	"log"
+	"os"
+	"time"
+)
+
+// doCrawl begins crawling the site at "homeurl"
+func doCrawl(homeurl string) itemSlice {
+	// set of what we have already crawled, our results
+	crawled := make(itemMap)
+
+	// a set of crawled URLs which have been cleaned
+	// useful so that we don't crawl http://a.com/index.html#about if we've
+	// already crawled http://a.com/index.html, or vice versa
+	crawledStripped := make(itemMap)
+
+	// channels to send work to and receive our results on
+	rxchan := make(chan *httpItem)
+	txchan := make(chan *httpItem)
+
+	// spin up our crawler workers
+	for i := 0; i < 10; i++ {
+		go crawlListner(txchan, rxchan)
+	}
+
+	// start a ticker which we'll use to output status and check for completion
+	ticker := time.Tick(250 * time.Millisecond)
+
+	// create first page's httpItem
+	homeitem, err := newHTTPItem(nil, homeurl)
+	if err != nil {
+		return nil
+	}
+
+	// set our number of outstanding pages (initially 1 to account for first page)
+	crawlingCount := 1
+
+	// start the home page crawl
+	crawled[homeitem.url.String()] = homeitem
+	crawledStripped[stripURL(homeitem.url)] = homeitem
+	txchan <- homeitem
+
+	// wait for results
+	for {
+		select {
+		case r := <-rxchan: // new results?
+			// add result to our results map
+			crawled[r.url.String()] = r
+
+			// decrease the outstanding page count by 1
+			crawlingCount--
+
+			// start crawly any new child pages we haven't yet crawled
+			for i, c := range r.children {
+				// see if we already have a result for this page
+				// if so, point to that result
+				if existing, ok := crawled[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
+				}
+
+				// see if we're already crawling this page (but maybe don't have results yet)
+				// if so, point to that item (we will have it as a result later!)
+				if existing, ok := crawled[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
+				}
+				if existing, ok := crawledStripped[stripURL(c.url)]; ok {
+					// we crawled a different version of this same page
+					// i.e. same page, different anchor. we don't need to crawl it again
+					// but we do need to keep it in our results, so copy the existing
+					// struct over for everything except the URLs
+					r.children[i].title = existing.title
+					r.children[i].linkType = existing.linkType
+					r.children[i].children = existing.children
+					continue
+				}
+
+				// haven't crawled this one yet, do so now
+				crawlingCount++
+				crawled[c.url.String()] = c
+				crawledStripped[stripURL(c.url)] = c
+				go func(newItem *httpItem) {
+					txchan <- newItem
+				}(c)
+			}
+
+		case <-ticker: // our regular ticker. for status output and checking for completion.
+			// output status to console
+			log.Printf("Crawled %v links, have %v left.\n", len(crawled), crawlingCount)
+
+			// see if we're finished
+			if crawlingCount == 0 {
+				// close our channels, signalling any workers to exit
+				close(txchan)
+				close(rxchan)
+
+				// finished! convert results map to a slice and return it
+				rslice := itemSlice{}
+				for _, v := range crawled {
+					rslice = append(rslice, v)
+				}
+				return rslice
+			}
+		}
+	}
+}
+
+// crawlListner is a for+select wrapper around crawlItem that listens
+// for new jobs and sends them off to crawlItem
+func crawlListner(txchan <-chan *httpItem, rxchan chan<- *httpItem) {
+	for {
+		select {
+		case newJob, ok := <-txchan:
+			if !ok {
+				// channel closed, finish
+				return
+			}
+			// perform the crawl
+			rxchan <- crawlItem(newJob)
+		}
+	}
+}
+
+// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
+// and filling out its structure as much as possible
+func crawlItem(item *httpItem) *httpItem {
+	// make sure this item is the same domain (i.e. URL "host part") as its referrer
+	if item.refurl != nil && item.url.Host != item.refurl.Host {
+		// skip URLs associated with other Hosts
+		item.linkType = tRemote
+		return item
+	}
+
+	// fetch page
+	text, err := fetchPage(item)
+	if err != nil {
+		return item
+	}
+
+	// parse links
+	title, links := parseLinks(text)
+	item.title = title
+
+	// walk links and add them as children to the current item
+	for _, l := range links {
+		newItem, err := newHTTPItem(item, l)
+		if err != nil {
+			continue // TODO bad item
+		}
+		item.children = append(item.children, newItem)
+	}
+
+	// send back our item struct now that it's all filled out
+	return item
+}
 
 func main() {
+	// see if we've got no arguments
+	if len(os.Args) == 1 {
+		fmt.Printf("\nD.O. Crawler 1.0  Copyright (c) 2015 Stephen Waits <steve@waits.net>  2015-02-17\n\n")
+		fmt.Printf("usage: %v <URLs...>\n\n", os.Args[0])
+		os.Exit(1)
+	}
+	// crawl each URL on the command line
+	for _, u := range os.Args[1:] {
+		pages := doCrawl(u)
+		if pages == nil {
+			log.Fatalf("unable to crawl %q, may be an invalid URL\n", u)
+		}
+		l := sitemapToLocations(pages)
+		j, _ := locationsToJSON(l)
+		fmt.Println(j)
+	}
 }

src/docrawler/docrawler_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -19,13 +18,13 @@ func TestMain(m *testing.M) {
 	// start our simple web server
 	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
 	if err != nil {
-		panic("unable to open port for http server") // TODO remove panic?
+		panic("unable to open port for http server")
 	}
 	http.Handle("/", http.FileServer(http.Dir("./testsite")))
 	go func() {
 		err = http.Serve(l, nil)
 		if err != nil {
-			panic("unable to start http server") // TODO remove panic?
+			panic("unable to start http server")
 		}
 	}()
 
@@ -43,8 +42,8 @@ func TestMain(m *testing.M) {
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error(err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal(err)
 	}
 }
 
@@ -52,21 +51,222 @@ func TestServerRunning(t *testing.T) {
 func TestBadRequest(t *testing.T) {
 	resp, err := http.Get(baseURL + "doesnotexist.html")
 	if resp.StatusCode != 404 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error("request for non existent file didn't fail properly")
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal("request for non existent file didn't fail properly")
 	}
 }
 
 // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
 func TestSimpleFetch(t *testing.T) {
-	body, err := fetch(baseURL + "fetch_test.html")
+	page, err := newHTTPItem(nil, baseURL+"fetch_test.html")
 	if err != nil {
-		t.Error(err)
+		t.Error("problem creating New httpItem struct")
+	}
+	body, err := fetchPage(page)
+	if err != nil {
+		t.Fatal(err)
 	}
 	desired := "<html><head></head><body></body></html>\n"
 	if body != desired {
-		fmt.Printf("   Got: '%v'\n", body)
-		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Logf("   Got: '%v'\n", body)
+		t.Logf("Wanted: '%v'\n", desired)
 		t.Error("fetch succeeded, but content mismatched")
 	}
 }
+
+// TestSimpleMap figures out the site map for the site in baseURL
+func TestSimpleMap(t *testing.T) {
+	pages := doCrawl(baseURL)
+
+	// because our crawl is non-deterministic, we have to do a complete
+	// cycle through every page, counting stuff, finding specific pages
+	// so that we can then verify certain aspects of each page
+	skipped, children := 0, 0
+	var homepage, aboutpage, skippage *httpItem
+	for _, p := range pages {
+		if p.title == "Home" {
+			homepage = p
+		} else if p.title == "About Test" {
+			aboutpage = p
+		} else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" {
+			skippage = p
+		}
+		if p.linkType == tRemote {
+			skipped++
+		}
+		children += len(p.children)
+	}
+
+	// check page counts
+	if len(pages) != 6 {
+		t.Logf("got %v, wanted %v\n", len(pages), 6)
+		t.Fatal("got wrong number of pages")
+	}
+	if children != 9 {
+		t.Logf("got %v, wanted %v\n", children, 8)
+		t.Fatal("got wrong number of total children")
+	}
+	if skipped != 1 {
+		t.Logf("got %v, wanted %v\n", skipped, 1)
+		t.Fatal("got wrong number of total skipped")
+	}
+
+	// check first page URL
+	if homepage.url.String() != baseURL {
+		t.Error("page URL is invalid")
+	}
+
+	// specific page number of children
+	if len(homepage.children) != 5 {
+		t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
+		t.Fatal("got wrong number of links")
+	}
+	if len(aboutpage.children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4)
+		t.Fatal("got wrong number of links")
+	}
+
+	// make sure remote pages were skipped
+	if homepage.linkType == tRemote {
+		t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String())
+		t.Error("child was skipped")
+	}
+	if skippage.linkType != tRemote {
+		t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String())
+		t.Error("child wasn't skipped")
+	}
+
+	// verify page content
+	// TODO find a better way to do this
+	//if homepage.children[0].url.String() != baseURL+"about.html" {
+	//t.Logf("   Got: %q\n", homepage.children[0].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"about.html")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.children[1].url.String() != baseURL+"assets/image.png" {
+	//t.Logf("   Got: %q\n", homepage.children[1].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.children[3].url.String() != baseURL+"scripts/blah.js" {
+	//t.Logf("   Got: %q\n", homepage.children[3].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+	//t.Error("link name is incorrect")
+	//}
+}
+
+// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
+func TestHeaderFetching(t *testing.T) {
+	basepage, err := newHTTPItem(nil, baseURL)
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err := newHTTPItem(basepage, "about.html")
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err = newHTTPItem(basepage, "assets/image.png")
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.linkType != tAsset {
+		t.Logf("got %q, wanted %q", page.linkType, tAsset)
+		t.Error("problem fetching filetype")
+	}
+}
+
+// TestJsonOutput gets a sitemap and then converts it to json
+func TestJsonOutput(t *testing.T) {
+	pages := doCrawl(baseURL)
+	l := sitemapToLocations(pages)
+	if len(l) != 2 {
+		t.Error("sitemapToLocations has the wrong number of locations")
+	}
+	if l[0].URL != baseURL {
+		t.Error("got an incorrect location")
+	}
+	if len(l[0].Links) != 1 {
+		t.Error("location 0 has wrong number of links")
+	}
+	if len(l[0].Assets) != 2 {
+		t.Logf("got %v", len(l[0].Assets))
+		t.Error("location 0 has wrong number of assets")
+	}
+	if len(l[0].Broken) != 1 {
+		t.Logf("got %v", len(l[0].Broken))
+		t.Error("location 0 has wrong number of broken urls")
+	}
+	if len(l[0].Remote) != 0 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 0 has wrong number of remote urls")
+	}
+	if l[1].URL != baseURL+"about.html" {
+		t.Error("got an incorrect location")
+	}
+	if len(l[1].Links) != 1 {
+		t.Error("location 1 has wrong number of links")
+	}
+	if len(l[1].Assets) != 2 {
+		t.Error("location 1 has wrong number of assets")
+	}
+	if len(l[1].Broken) != 0 {
+		t.Logf("got %v", len(l[1].Broken))
+		t.Error("location 1 has wrong number of broken urls")
+	}
+	if len(l[1].Remote) != 1 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 1 has wrong number of remote urls")
+	}
+
+	// this is a cheater test because the output is from a run of the code being
+	// test itself. but, it's been examined and I think it's right. and, there's not
+	// many other ways to test this without some significant pain
+	cheaterTest := `[
+  {
+    "URL": "http://localhost:8765/",
+    "Title": "Home",
+    "Links": [
+      "http://localhost:8765/about.html"
+    ],
+    "Assets": [
+      "http://localhost:8765/assets/image.png",
+      "http://localhost:8765/scripts/blah.js"
+    ],
+    "Broken": [
+      "http://localhost:8765/zzzbroken.html"
+    ],
+    "Remote": null
+  },
+  {
+    "URL": "http://localhost:8765/about.html",
+    "Title": "About Test",
+    "Links": [
+      "http://localhost:8765/"
+    ],
+    "Assets": [
+      "http://localhost:8765/assets/image.png",
+      "http://localhost:8765/scripts/blah.js"
+    ],
+    "Broken": null,
+    "Remote": [
+      "http://doesntexist23492387492837492374982734.com/"
+    ]
+  }
+]`
+	j, err := locationsToJSON(l)
+	if err != nil {
+		t.Error("locationsToJSON failed")
+	}
+	if j != cheaterTest {
+		t.Error("locationsToJSON text doesn't match")
+	}
+	t.Log(j)
+}

src/docrawler/fetch.go

@@ -1,18 +1,87 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 )
 
-// fetch takes a URL, GETs it, and returns the body as a string
-func fetch(url string) (string, error) {
-	// GET the url
-	resp, err := http.Get(url)
+// custom errors
+var (
+	errContentTypeNotFound = errors.New("no Content-Type header found")
+	errFetchError          = errors.New("couldn't fetch item")
+	errFileTypeUnknown     = errors.New("couldn't determine file type")
+)
+
+// fetchFiletype performs an http HEAD to get the media type, and sets it
+// directly in httpItem.mediaType
+func fetchFiletype(item *httpItem) error {
+	resp, err := http.Head(item.url.String())
+	if err != nil {
+		item.linkType = tBroken
+		return err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		item.linkType = tBroken
+		return errFetchError
+	}
+
+	// pull the content type out of the http header
+	contentType, ok := resp.Header["Content-Type"]
+	if !ok {
+		// no Content-Type header found
+		item.linkType = tBroken
+		return errContentTypeNotFound
+	}
+
+	// parse mime type
+	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		item.linkType = tBroken
+		return err
+	}
+
+	// success, set item type and return
+	if mediatype == "text/html" {
+		item.linkType = tHTMLPage
+	} else {
+		item.linkType = tAsset
+	}
+	return nil
+}
+
+// fetchPage takes a httpItem, GETs it, and returns the body as a string
+func fetchPage(item *httpItem) (string, error) {
+	// figure out the file type
+	err := fetchFiletype(item)
 	if err != nil {
 		return "", err
 	}
 
+	// make sure we got a file type!
+	if item.linkType == tUnknown {
+		return "", errFileTypeUnknown
+	}
+
+	// we only want to fetch html, so return if it's anything else
+	if item.linkType != tHTMLPage {
+		return "", nil // but this isn't an error!
+	}
+
+	// GET the url
+	resp, err := http.Get(item.url.String())
+	if err != nil {
+		return "", err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return "", errFetchError
+	}
+
 	// read the body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {

src/docrawler/fetch_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedHeaderFetching(t *testing.T) {
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
+	}
+}
+
+// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedFetching(t *testing.T) {
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if _, err := fetchPage(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
+	}
+}

src/docrawler/item.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"net/url"
+)
+
+// itemSlice is a convenience type for a slice of items
+type itemSlice []*httpItem
+
+// itemMap convenience type for map of string's (URL) to items
+type itemMap map[string]*httpItem
+
+// itemType is an enum so we know how to classify each item
+type itemType int
+
+// enums for itemType
+const (
+	tUnknown itemType = iota
+	tHTMLPage
+	tAsset
+	tRemote
+	tBroken
+)
+
+// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+type httpItem struct {
+	url      *url.URL
+	refurl   *url.URL
+	title    string
+	linkType itemType
+	children itemSlice
+}
+
+// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{}
+func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.url.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// figure out referrer url
+	var rurl *url.URL
+	if referrer != nil {
+		rurl = referrer.url
+	}
+
+	// create struct and return
+	return &httpItem{url: u, refurl: rurl}, nil
+}

src/docrawler/item_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures
+func TestBadnewHTTPItem(t *testing.T) {
+	_, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	if err == nil {
+		t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error")
+	}
+}

src/docrawler/output.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc.
+type Location struct {
+	URL    string
+	Title  string
+	Links  []string
+	Assets []string
+	Broken []string
+	Remote []string
+}
+
+// implement Location slice sorting (by URL)
+type byURL []*Location
+
+// implement Sort interface on byURL (which is a []*Location)
+func (l byURL) Len() int           { return len(l) }
+func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL }
+func (l byURL) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
+
+// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling
+func sitemapToLocations(pages itemSlice) []*Location {
+	// build a map of our pages
+	pageMap := make(map[string]*httpItem)
+	for _, p := range pages {
+		pageMap[p.url.String()] = p
+	}
+
+	// build a slice of locations (one per page)
+	var locations []*Location
+	for _, p := range pages {
+		if p.linkType == tHTMLPage {
+			// create a location for this page
+			l := &Location{URL: p.url.String(), Title: p.title}
+
+			// add its children
+			for _, c := range p.children {
+				// look up this child's media type from the root list of pages
+				//mediaType := pageMap[c.url.String()].mediaType
+				if c.linkType == tRemote {
+					l.Remote = append(l.Remote, c.url.String())
+				} else if c.linkType == tHTMLPage {
+					l.Links = append(l.Links, c.url.String())
+				} else if c.linkType == tBroken {
+					l.Broken = append(l.Broken, c.url.String())
+				} else if c.linkType == tAsset {
+					l.Assets = append(l.Assets, c.url.String())
+				} else {
+					// unknown link here, which means it failed to crawl, let's call it "broken"
+					l.Broken = append(l.Broken, c.url.String())
+				}
+
+			}
+
+			// now uniq & sort the children slices
+			l.Remote = uniqStrings(l.Remote)
+			l.Links = uniqStrings(l.Links)
+			l.Broken = uniqStrings(l.Broken)
+			l.Assets = uniqStrings(l.Assets)
+			sort.Strings(l.Remote)
+			sort.Strings(l.Links)
+			sort.Strings(l.Broken)
+			sort.Strings(l.Assets)
+
+			// and add this location to our slice
+			locations = append(locations, l)
+		}
+	}
+
+	// sort the Locations themselves and return
+	sort.Sort(byURL(locations))
+	return locations
+}
+
+// locationsToJSON takes a *Location slice and marshals it into a JSON string
+func locationsToJSON(locations []*Location) (string, error) {
+	b, err := json.MarshalIndent(locations, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

src/docrawler/parse.go

@@ -2,20 +2,28 @@ package main
 
 import (
 	"regexp"
-	"sort"
+	"strings"
 )
 
 // a regexp which captures src/href/xhref="..." from text
-var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice
+var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html links out of it,
-// and returns a sorted slice of the captures found
-func parse(s string) ([]string, error) {
-	// find all of the matches (submatch = include capture groups)
+// parseLinks takes a string and attempts to parse any html title and all links out of it,
+// and returns a slice of the captures found
+func parseLinks(s string) (string, []string) {
+	// find the title, or default to empty
+	title := ""
+	titleMatches := reTitle.FindStringSubmatch(s)
+	if titleMatches != nil {
+		// we found a title, replace our default title with the result
+		title = strings.TrimSpace(titleMatches[1])
+	}
+
+	// find all of the URL matches (submatch = include capture groups)
 	results := []string{}
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	sort.Strings(results)
-	return results, nil
+	return title, results
 }

src/docrawler/parse_test.go

@@ -4,27 +4,32 @@ import (
 	"testing"
 )
 
-// TestSimpleParse verifies that we can extract URLs (etc.) from a known document
+// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document
 func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
+		<TITle> Test Page  </title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	matches, err := parse(doc)
-	if err != nil {
-		t.Error(err)
+	title, matches := parseLinks(doc)
+	if title != "Test Page" {
+		t.Error("got wrong title")
 	}
-	if len(matches) != 2 {
+	if len(matches) != 3 {
 		t.Error("invalid number of matches in parse")
 	}
-	if matches[0] != "/about.html" {
+	if matches[0] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
-	if matches[1] != "/assets/image.png" {
+	if matches[1] != "/about.html" {
+		t.Error("match text is invalid")
+	}
+	if matches[2] != "scripts/blah.js" {
 		t.Error("match text is invalid")
 	}
 }

src/docrawler/testsite/about.html

@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>About Test</title>
+		<script src="scripts/blah.js"/>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/">fooey!</a>
+		<a href="http://doesntexist23492387492837492374982734.com/">bad one</a>
+	</body>
+</html>

src/docrawler/testsite/index.html

@@ -1,8 +1,12 @@
 <html>
 	<head>
+		<title>Home</title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<a href="/about.html">duplicate!</a>
+		<script src="scripts/blah.js"/>
+		<a href="/zzzbroken.html">oops!</a>
 	</body>
 </html>

src/docrawler/testsite/scripts/blah.js

@@ -0,0 +1 @@
+

src/docrawler/url.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+// custom errors
+var (
+	errInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return errInvalidURL
+	}
+	return nil // success!
+}
+
+// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
+	if err != nil {
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
+
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
+}
+
+// stripURL returns a version of the URL without the "Fragment" part,
+// which is anything after the '#' character, so:
+// http://a.com/blah.html#anchor becomes http://a.com/blah.html
+// ... and without files like "index.htm" in the Path (since that's
+// equivalent to "/"
+func stripURL(u *url.URL) string {
+	// copy the url locally
+	ucopy := *u
+
+	// blank out the anchor, return URI string
+	ucopy.Fragment = ""
+
+	// remove index.html if it exists
+	suffixes := []string{"index.html", "index.htm"}
+	for _, s := range suffixes {
+		if strings.HasSuffix(strings.ToLower(ucopy.Path), s) {
+			ucopy.Path = ucopy.Path[0 : len(ucopy.Path)-len(s)]
+		}
+	}
+
+	// return new stripped down URL string
+	return ucopy.String()
+}

src/docrawler/url_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		t.Logf("   Got: %q\n", u.Host)
+		t.Logf("Wanted: %q\n", "somehost.com:8765")
+		t.Error("host extraction failed")
+	}
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := resolveURL(referrer, current)
+	if err != nil {
+		t.Error("resolveURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		t.Logf("   Got: %q\n", u)
+		t.Logf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
+}
+
+// TestBadResolve tries to resolve bogus URLs to be sure we're getting errors
+/* TODO not sure how to make url.Parse fail in these cases!!!
+func TestBadResolve(t* testing.T) {
+	u, err := resolveURL("http://a.com/", "h%20t\tp://doesnte%20xist23492387492837492374982734.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+	u, err = resolveURL("h%20t\tp://doesnte%20xist23492387492837492374982734.com/", "http://a.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+}
+*/
+
+// TestURLStripAnchor tests the stripURL() function
+func TestURLStripAnchor(t *testing.T) {
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/index.html?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+	{
+		u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/index.htm?x=y#foo")
+		if err != nil {
+			t.Fatal("couldn't parse a URL")
+		}
+		stripped := stripURL(u)
+		if stripped != "http://swaits:pass@somehost.com:8765/blah/?x=y" {
+			t.Error("stripping anchor #foo from URL failed")
+		}
+	}
+}

src/docrawler/util.go

@@ -0,0 +1,23 @@
+package main
+
+// uniqStrings takes a slice of strings and removes any duplicates
+// note: does not guarantee any order (or stability of order)
+func uniqStrings(strs []string) []string {
+	// handle special case of a nil parameter
+	if strs == nil {
+		return nil
+	}
+
+	// add each string to the slice as keys (used like a "set")
+	set := make(map[string]struct{})
+	for _, s := range strs {
+		set[s] = struct{}{}
+	}
+
+	// now copy all the keys over to a new slice and return it
+	result := []string{}
+	for key := range set {
+		result = append(result, key)
+	}
+	return result
+}

src/docrawler/util_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"sort"
+	"testing"
+)
+
+// TestUniqStrings passes a slice of strings to uniqStrings() to see if it removes
+// all duplicates
+func TestUniqStrings(t *testing.T) {
+	strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"}
+	uniq := uniqStrings(strs)
+	if len(uniq) != 5 {
+		t.Error("wrong number of strings after uniq'ing")
+	}
+	sort.Strings(uniq)
+	if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" {
+		t.Error("uniq values are in correct!")
+	}
+}
+
+// TestUniqStringsNilParm passes a nil slice to uniqStrings to be sure it returns nil
+func TestUniqStringsNilParm(t *testing.T) {
+	uniq := uniqStrings(nil)
+	if uniq != nil {
+		t.Error("uniqStrings should return a nil slice when passed a nil slice")
+	}
+}

util/testfilter.sh

@@ -2,6 +2,6 @@
 
 # colorize the output of "go test"
 
-sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
+sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
 	sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/''   | \
 	sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/''