Merge from develop. First version satisfying all basic requirements.

.hgignore

@@ -1,2 +1,3 @@
 syntax: glob
-bin
+bin
+cover.out

Makefile

@@ -3,18 +3,20 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest run clean env vendor
+.PHONY: build fmt lint vet test autotest cover doc run clean env stats vendor
 
 # configuration
 APPNAME := docrawler
 VENDOR := .vendor
 PACKAGES :=                     \
+	golang.org/x/tools/cmd/cover  \
+	golang.org/x/tools/cmd/godoc  \
 	golang.org/x/tools/cmd/vet    \
 	github.com/golang/lint/golint
 
 # override GOPATH to be our vendor directory
-export GOPATH := ${PWD}/${VENDOR}
-export PATH := ${PWD}/${VENDOR}/bin:${PATH}
+export GOPATH := ${PWD}/${VENDOR}:${PWD}
+export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH}
 
 default: build
 
@@ -24,35 +26,54 @@ build: fmt lint vet
 fmt:
 	go fmt ./src/...
 
+# TODO golint is not accepting the "./src/..." for some reason (from within make)
 lint:
-	golint ./src
+	golint ./src/${APPNAME}/*.go
 
 vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./testfilter.sh
+	go test -v ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \
-	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./testfilter.sh'
+		xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+
+cover:
+	go test -coverprofile cover.out ./src/...  && go tool cover -html cover.out
+
+doc:
+	godoc -http=:6060 -index
 
 run: build
 	./bin/${APPNAME}
 
 clean:
-	rm -f ./bin/${APPNAME}
+	rm -f ./bin/${APPNAME} ./cover.out
 
 env:
 	go env
 
+stats:
+	@echo ===============================================================================
+	@echo Tests
+	@echo ===============================================================================
+	@cloc --match-f=_test.go ./src
+	@echo
+	@echo
+	@echo ===============================================================================
+	@echo Everything Else
+	@echo ===============================================================================
+	@cloc --not-match-f=_test.go ./src
+
 # vendor
 # * wipes our vendor directory completely
-# * fetches all packages in ${PACKAGES}
+# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)
 # * removes source control dirs so packages are vendorable (commitable)
 vendor:
-	rm -dRf ./${VENDOR}/src                          && \
-	go get -u ${PACKAGES}                            && \
+	rm -dRf ./${VENDOR}                              && \
+	GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES}    && \
 	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
 	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
 	rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \

docs/Design.md

@@ -39,11 +39,11 @@ These are features I'd like to include.
 
 * use channels to form a pipeline
 * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines)
+* pipeline may allows us to scale concurrency at the bottlenecks
 * could use Interfaces for things like Parser, Getter, but I think best avoided until needed
 * launch goroutine per fetch
 * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it
 
-
 ## Wishlist
 
 These are features that sound cool but are probably out of scope.

docs/TODO.md

@@ -0,0 +1,22 @@
+# To Do
+
+## Bugs
+
+* deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.)
+  * should only crawl these pages once
+  * but should be included in the links part
+* when sorting links, "uniq" them as well
+
+## Cleanup
+
+* rename Page -> crawlerState
+* helper function to do the whole crawl process
+* document code better
+  * test with godoc
+
+## Features
+
+* some sort of status during crawl?
+* command line options, flag parsing
+* check RFCs on URL character sets, compliance, etc.
+* packagize parts of app

src/docrawler/docrawler.go

@@ -1,6 +1,93 @@
 package main
 
-import ()
+import (
+	"fmt"
+	"os"
+)
+
+// docrawl begins crawling the site at "url"
+func docrawl(url string) []*Page {
+	// our result structure
+	var sitemap []*Page
+
+	// what we've already parsed, what we still need to parse
+	crawled := make(map[string]*Page)
+	queued := make(map[string]*Page)
+
+	// create first page and add to the queue
+	homepage, _ := NewPage(nil, url) // TODO check error
+	queued[homepage.URL.String()] = homepage
+
+	// main loop
+	for len(queued) > 0 {
+		// pop a page off of queued
+		var dest *Page
+		var destURL string
+		for destURL, dest = range queued {
+			break
+		}
+		delete(queued, destURL)
+
+		// mark this page as complete
+		crawled[dest.URL.String()] = dest
+
+		// add page to list
+		sitemap = append(sitemap, dest)
+
+		// make sure this URL has the same hostname as our first page
+		if dest.URL.Host != homepage.URL.Host {
+			// skip URLs associated with other Hosts
+			dest.Skipped = true
+			continue
+		}
+
+		// fetch page
+		text, err := fetchPage(dest)
+		if err != nil {
+			dest.Broken = true
+			continue
+		}
+
+		// parse links
+		title, links, err := parseLinks(text)
+		if err != nil {
+			dest.Broken = true
+			continue
+		}
+		dest.Title = title
+		for _, l := range links {
+			p, err := NewPage(dest, l)
+			if err != nil {
+				continue // TODO check error
+			}
+
+			// see if we already know about this page
+			pCrawled, haveCrawled := crawled[p.URL.String()]
+			pQueued, haveQueued := queued[p.URL.String()]
+			if haveCrawled {
+				// add the previously crawled page to children
+				dest.Children = append(dest.Children, pCrawled)
+			} else if haveQueued {
+				// add the previously queued page to children
+				dest.Children = append(dest.Children, pQueued)
+			} else {
+				// add this new page (never seen) to children
+				dest.Children = append(dest.Children, p)
+				// .. and queue it up for crawling
+				queued[p.URL.String()] = p
+			}
+		}
+	}
+
+	return sitemap
+}
 
 func main() {
+	// crawl each URL on the command line
+	for _, u := range os.Args[1:] {
+		pages := docrawl(u)
+		l := sitemapToLocations(pages)
+		j, _ := locationsToJSON(l)
+		fmt.Println(j)
+	}
 }

src/docrawler/docrawler_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -19,13 +18,13 @@ func TestMain(m *testing.M) {
 	// start our simple web server
 	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
 	if err != nil {
-		panic("unable to open port for http server") // TODO remove panic?
+		panic("unable to open port for http server")
 	}
 	http.Handle("/", http.FileServer(http.Dir("./testsite")))
 	go func() {
 		err = http.Serve(l, nil)
 		if err != nil {
-			panic("unable to start http server") // TODO remove panic?
+			panic("unable to start http server")
 		}
 	}()
 
@@ -43,8 +42,8 @@ func TestMain(m *testing.M) {
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error(err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal(err)
 	}
 }
 
@@ -52,46 +51,191 @@ func TestServerRunning(t *testing.T) {
 func TestBadRequest(t *testing.T) {
 	resp, err := http.Get(baseURL + "doesnotexist.html")
 	if resp.StatusCode != 404 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error("request for non existent file didn't fail properly")
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal("request for non existent file didn't fail properly")
 	}
 }
 
-// TestFetch makes sure we can fetch a file and we get exactly what we expect
-func TestFetch(t *testing.T) {
-	body, err := fetch(baseURL + "fetch_test.html")
+// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
+func TestSimpleFetch(t *testing.T) {
+	page, err := NewPage(nil, baseURL+"fetch_test.html")
 	if err != nil {
-		t.Error(err)
+		t.Error("problem creating New Page struct")
+	}
+	body, err := fetchPage(page)
+	if err != nil {
+		t.Fatal(err)
 	}
 	desired := "<html><head></head><body></body></html>\n"
 	if body != desired {
-		fmt.Printf("   Got: '%v'\n", body)
-		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Logf("   Got: '%v'\n", body)
+		t.Logf("Wanted: '%v'\n", desired)
 		t.Error("fetch succeeded, but content mismatched")
 	}
 }
 
-// TestParse verifies that we can extract URLs (etc.) from a known document
-func TestParse(t *testing.T) {
-	doc := `<html>
-	<head>
-	</head>
-	<body>
-		<img src="/assets/image.png"/>
-		<a href="/about.html">
-	</body>
-</html>`
-	matches, err := parse(doc)
+// TestSimpleMap figures out the site map for the site in baseURL
+func TestSimpleMap(t *testing.T) {
+	pages := docrawl(baseURL)
+
+	// because our crawl is non-deterministic, we have to do a complete
+	// cycle through every page, counting stuff, finding specific pages
+	// so that we can then verify certain aspects of each page
+	skipped, children := 0, 0
+	var homepage, aboutpage, skippage *Page
+	for _, p := range pages {
+		if p.Title == "Home" {
+			homepage = p
+		} else if p.Title == "About Test" {
+			aboutpage = p
+		} else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" {
+			skippage = p
+		}
+		if p.Skipped {
+			skipped++
+		}
+		children += len(p.Children)
+	}
+
+	// check page counts
+	if len(pages) != 6 {
+		t.Logf("got %v, wanted %v\n", len(pages), 6)
+		t.Fatal("got wrong number of pages")
+	}
+	if children != 8 {
+		t.Logf("got %v, wanted %v\n", children, 8)
+		t.Fatal("got wrong number of total children")
+	}
+	if skipped != 1 {
+		t.Logf("got %v, wanted %v\n", skipped, 1)
+		t.Fatal("got wrong number of total skipped")
+	}
+
+	// check first page URL
+	if homepage.URL.String() != baseURL {
+		t.Error("page URL is invalid")
+	}
+
+	// specific page number of children
+	if len(homepage.Children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(homepage.Children), 4)
+		t.Fatal("got wrong number of links")
+	}
+	if len(aboutpage.Children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4)
+		t.Fatal("got wrong number of links")
+	}
+
+	// make sure remote pages were skipped
+	if homepage.Skipped != false {
+		t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String())
+		t.Error("child was skipped")
+	}
+	if skippage.Skipped != true {
+		t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String())
+		t.Error("child wasn't skipped")
+	}
+
+	// verify page content
+	// TODO find a better way to do this
+	//if homepage.Children[0].URL.String() != baseURL+"about.html" {
+	//t.Logf("   Got: %q\n", homepage.Children[0].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"about.html")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.Children[1].URL.String() != baseURL+"assets/image.png" {
+	//t.Logf("   Got: %q\n", homepage.Children[1].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" {
+	//t.Logf("   Got: %q\n", homepage.Children[3].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+	//t.Error("link name is incorrect")
+	//}
+}
+
+// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
+func TestHeaderFetching(t *testing.T) {
+	basepage, err := NewPage(nil, baseURL)
 	if err != nil {
-		t.Error(err)
+		t.Error("problem creating New Page struct")
 	}
-	if len(matches) != 2 {
-		t.Error("invalid number of matches in parse")
+	if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" {
+		t.Error("problem fetching filetype")
 	}
-	if matches[0] != "/assets/image.png" {
-		t.Error("match text is invalid")
+
+	page, err := NewPage(basepage, "about.html")
+	if err != nil {
+		t.Error("problem creating New Page struct")
 	}
-	if matches[1] != "/about.html" {
-		t.Error("match text is invalid")
+	if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err = NewPage(basepage, "assets/image.png")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" {
+		t.Logf("got %q, wanted %q", page.MediaType, "image/png")
+		t.Error("problem fetching filetype")
 	}
 }
+
+// TestJsonOutput gets a sitemap and then converts it to json
+func TestJsonOutput(t *testing.T) {
+	pages := docrawl(baseURL)
+	l := sitemapToLocations(pages)
+	if len(l) != 2 {
+		t.Error("sitemapToLocations has the wrong number of locations")
+	}
+	if l[0].URL != baseURL {
+		t.Error("got an incorrect location")
+	}
+	if len(l[0].Links) != 1 {
+		t.Error("location 0 has wrong number of links")
+	}
+	if len(l[0].Assets) != 2 {
+		t.Logf("got %v", len(l[0].Assets))
+		t.Error("location 0 has wrong number of assets")
+	}
+	if len(l[0].Broken) != 1 {
+		t.Logf("got %v", len(l[0].Broken))
+		t.Error("location 0 has wrong number of broken urls")
+	}
+	if len(l[0].Remote) != 0 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 0 has wrong number of remote urls")
+	}
+	if l[1].URL != baseURL+"about.html" {
+		t.Error("got an incorrect location")
+	}
+	if len(l[1].Links) != 1 {
+		t.Error("location 1 has wrong number of links")
+	}
+	if len(l[1].Assets) != 2 {
+		t.Error("location 1 has wrong number of assets")
+	}
+	if len(l[1].Broken) != 0 {
+		t.Logf("got %v", len(l[1].Broken))
+		t.Error("location 1 has wrong number of broken urls")
+	}
+	if len(l[1].Remote) != 1 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 1 has wrong number of remote urls")
+	}
+
+	// this is a cheater test because the output is from a run of the code being
+	// test itself. but, it's been examined and I think it's right. and, there's not
+	// many other ways to test this without some significant pain
+	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]`
+	j, err := locationsToJSON(l)
+	if err != nil {
+		t.Error("locationsToJSON failed")
+	}
+	if j != cheaterTest {
+		t.Error("locationsToJSON text doesn't match")
+	}
+	t.Log(j)
+}

src/docrawler/fetch.go

@@ -1,22 +1,79 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 )
 
-func fetch(url string) (string, error) {
-	// GET the url
-	resp, err := http.Get(url)
+var (
+	errContentTypeNotFound = errors.New("no Content-Type header found")
+	errFetchError          = errors.New("couldn't fetch page")
+)
+
+// fetchFiletype performs an http HEAD to get the media type, and sets it
+// directly in Page.MediaType
+func fetchFiletype(page *Page) error {
+	resp, err := http.Head(page.URL.String())
+	if err != nil {
+		return err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return errFetchError
+	}
+
+	// pull the content type out of the http header
+	contentType, ok := resp.Header["Content-Type"]
+	if !ok {
+		// no Content-Type header found
+		return errContentTypeNotFound
+	}
+
+	// parse mime type
+	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		return err
+	}
+
+	// set and return
+	page.MediaType = mediatype
+	return nil
+}
+
+// fetchPage takes a Page, GETs it, and returns the body as a string
+func fetchPage(page *Page) (string, error) {
+	// figure out the file type
+	err := fetchFiletype(page)
 	if err != nil {
 		return "", err
 	}
 
+	// we only want to fetch html
+	if page.MediaType != "text/html" {
+		return "", nil // but this isn't an error!
+	}
+
+	// GET the url
+	resp, err := http.Get(page.URL.String())
+	if err != nil {
+		return "", err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return "", errFetchError
+	}
+
 	// read the body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
 	}
+
+	// success! close the body, return it as a string
 	resp.Body.Close()
 	return string(body), nil
 }

src/docrawler/fetch_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedHeaderFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
+	}
+}
+
+// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if _, err := fetchPage(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
+	}
+}

src/docrawler/output.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc.
+type Location struct {
+	URL    string
+	Title  string
+	Links  []string
+	Assets []string
+	Broken []string
+	Remote []string
+}
+
+func sitemapToLocations(pages []*Page) []*Location {
+	// build a map of our pages
+	pageMap := make(map[string]*Page)
+	for _, p := range pages {
+		pageMap[p.URL.String()] = p
+	}
+
+	// build a slice of locations (one per page)
+	var locations []*Location
+	for _, p := range pages {
+		if p.MediaType == "text/html" {
+			// create a location for this page
+			l := &Location{URL: p.URL.String(), Title: p.Title}
+
+			// add its children
+			for _, c := range p.Children {
+				// look up this child's media type from the root list of pages
+				//mediaType := pageMap[c.URL.String()].MediaType
+				if c.Skipped {
+					l.Remote = append(l.Remote, c.URL.String())
+				} else if c.MediaType == "text/html" {
+					l.Links = append(l.Links, c.URL.String())
+				} else if c.Broken {
+					l.Broken = append(l.Broken, c.URL.String())
+				} else {
+					l.Assets = append(l.Assets, c.URL.String())
+				}
+			}
+
+			// now sort the children slices
+			sort.Strings(l.Remote)
+			sort.Strings(l.Links)
+			sort.Strings(l.Broken)
+			sort.Strings(l.Assets)
+
+			// and add this location to our slice
+			locations = append(locations, l)
+		}
+	}
+
+	return locations
+}
+
+func locationsToJSON(locations []*Location) (string, error) {
+	b, err := json.Marshal(locations)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

src/docrawler/page.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"net/url"
+)
+
+// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+type Page struct {
+	URL       *url.URL
+	Title     string
+	MediaType string
+	Skipped   bool
+	Broken    bool
+	Children  []*Page
+}
+
+// NewPage takes a referring Page + a URL and returns a new &Page{}
+func NewPage(referrer *Page, rawurl string) (*Page, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.URL.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// create struct and return
+	return &Page{URL: u, Skipped: false, Broken: false}, nil
+}

src/docrawler/page_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestBadNewPage tests NewPage to see if we are getting expected failures
+func TestBadNewPage(t *testing.T) {
+	_, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	if err == nil {
+		t.Error("tried creating a NewPage with an invalid URL but didn't get an error")
+	}
+}

src/docrawler/parse.go

@@ -2,16 +2,28 @@ package main
 
 import (
 	"regexp"
+	"strings"
 )
 
-func parse(s string) ([]string, error) {
-	// hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.)
-	re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`)
+// a regexp which captures src/href/xhref="..." from text
+var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-	// find all of the matches (submatch = include capture groups)
+// parseLinks takes a string and attempts to parse any html title and all links out of it,
+// and returns a slice of the captures found
+func parseLinks(s string) (string, []string, error) {
+	// find the title, or default to empty
+	title := ""
+	titleMatches := reTitle.FindStringSubmatch(s)
+	if titleMatches != nil {
+		// we found a title, replace our default title with the result
+		title = strings.TrimSpace(titleMatches[1])
+	}
+
+	// find all of the URL matches (submatch = include capture groups)
 	results := []string{}
-	for _, match := range re.FindAllStringSubmatch(s, -1) {
+	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	return results, nil
+	return title, results, nil
 }

src/docrawler/parse_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document
+func TestSimpleParse(t *testing.T) {
+	doc := `<html>
+	<head>
+		<TITle> Test Page  </title>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/about.html">fooey!</a>
+		<script href="scripts/blah.js"/>
+	</body>
+</html>`
+	title, matches, err := parseLinks(doc)
+	if err != nil {
+		t.Error(err)
+	}
+	if title != "Test Page" {
+		t.Error("got wrong title")
+	}
+	if len(matches) != 3 {
+		t.Error("invalid number of matches in parse")
+	}
+	if matches[0] != "/assets/image.png" {
+		t.Error("match text is invalid")
+	}
+	if matches[1] != "/about.html" {
+		t.Error("match text is invalid")
+	}
+	if matches[2] != "scripts/blah.js" {
+		t.Error("match text is invalid")
+	}
+}

src/docrawler/testsite/about.html

@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>About Test</title>
+		<script src="scripts/blah.js"/>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/">fooey!</a>
+		<a href="http://doesntexist23492387492837492374982734.com/">bad one</a>
+	</body>
+</html>

src/docrawler/testsite/index.html

@@ -1,8 +1,11 @@
 <html>
 	<head>
+		<title>Home</title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script src="scripts/blah.js"/>
+		<a href="/zzzbroken.html">oops!</a>
 	</body>
 </html>

src/docrawler/testsite/scripts/blah.js

@@ -0,0 +1 @@
+

src/docrawler/url.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+var (
+	errInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return errInvalidURL
+	}
+	return nil // success!
+}
+
+// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
+	if err != nil {
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
+
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
+}

src/docrawler/url_test.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		t.Logf("   Got: %q\n", u.Host)
+		t.Logf("Wanted: %q\n", "somehost.com:8765")
+		t.Error("host extraction failed")
+	}
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := resolveURL(referrer, current)
+	if err != nil {
+		t.Error("resolveURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		t.Logf("   Got: %q\n", u)
+		t.Logf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
+}
+
+// TestBadResolve tries to resolve bogus URLs to be sure we're getting errors
+/* TODO not sure how to make url.Parse fail in these cases!!!
+func TestBadResolve(t* testing.T) {
+	u, err := resolveURL("http://a.com/", "h%20t\tp://doesnte%20xist23492387492837492374982734.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+	u, err = resolveURL("h%20t\tp://doesnte%20xist23492387492837492374982734.com/", "http://a.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+}
+*/