flow: Closed <feature> 'remove_error_from_parselinks'.

* crawled is now a map to httpItem's (instead of struct{}) so we can connect pages mid-crawl
* added duplicate link to html site, test updated to match
* now any httpItem with linkType == tUnknown is listed as "Broken" in the output

.hgignore

@@ -1,2 +1,3 @@
 syntax: glob
-bin
+bin
+cover.out

Makefile

@@ -3,18 +3,20 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest run clean env vendor
+.PHONY: build fmt lint vet test autotest cover doc run clean env stats vendor
 
 # configuration
 APPNAME := docrawler
 VENDOR := .vendor
 PACKAGES :=                     \
+	golang.org/x/tools/cmd/cover  \
+	golang.org/x/tools/cmd/godoc  \
 	golang.org/x/tools/cmd/vet    \
 	github.com/golang/lint/golint
 
 # override GOPATH to be our vendor directory
-export GOPATH := ${PWD}/${VENDOR}
-export PATH := ${PWD}/${VENDOR}/bin:${PATH}
+export GOPATH := ${PWD}/${VENDOR}:${PWD}
+export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH}
 
 default: build
 
@@ -24,8 +26,9 @@ build: fmt lint vet
 fmt:
 	go fmt ./src/...
 
+# TODO golint is not accepting the "./src/..." for some reason (from within make)
 lint:
-	golint ./src
+	golint ./src/${APPNAME}/*.go
 
 vet:
 	go vet ./src/...
@@ -35,24 +38,42 @@ test:
 
 autotest:
 	fswatch -or ./src | \
-	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+		xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+
+cover:
+	go test -coverprofile cover.out ./src/...  && go tool cover -html cover.out
+
+doc:
+	godoc -http=:6060 -index
 
 run: build
 	./bin/${APPNAME}
 
 clean:
-	rm -f ./bin/${APPNAME}
+	rm -f ./bin/${APPNAME} ./cover.out
 
 env:
 	go env
 
+stats:
+	@echo ===============================================================================
+	@echo Tests
+	@echo ===============================================================================
+	@cloc --match-f=_test.go ./src
+	@echo
+	@echo
+	@echo ===============================================================================
+	@echo Everything Else
+	@echo ===============================================================================
+	@cloc --not-match-f=_test.go ./src
+
 # vendor
 # * wipes our vendor directory completely
-# * fetches all packages in ${PACKAGES}
+# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)
 # * removes source control dirs so packages are vendorable (commitable)
 vendor:
-	rm -dRf ./${VENDOR}/src                          && \
-	go get -u ${PACKAGES}                            && \
+	rm -dRf ./${VENDOR}                              && \
+	GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES}    && \
 	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
 	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
 	rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \

docs/Design.md

@@ -44,7 +44,6 @@ These are features I'd like to include.
 * launch goroutine per fetch
 * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it
 
-
 ## Wishlist
 
 These are features that sound cool but are probably out of scope.

docs/TODO.md

@@ -0,0 +1,21 @@
+# To Do
+
+## Bugs
+
+* deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.)
+  * should only crawl these pages once
+  * but should be included in the links part
+* be smart about index.html?
+
+## Cleanup
+
+* helper function to do the whole crawl process
+* document code better
+  * test with godoc
+* omit fields in json when nil? (Broken, etc.)
+
+## Features
+
+* command line options, flag parsing
+* check RFCs on URL character sets, compliance, etc.
+* packagize parts of app

src/docrawler/docrawler.go

@@ -1,6 +1,134 @@
 package main
 
-import ()
+import (
+	"fmt"
+	"log"
+	"os"
+	"time"
+)
+
+// docrawl begins crawling the site at "homeurl"
+func doCrawl(homeurl string) itemSlice {
+	// set of what we have already crawled
+	crawled := make(itemMap)
+
+	// map (of urlString -> httpItem) of our results
+	results := make(itemMap)
+
+	// a channel to receive our results on
+	rchan := make(chan *httpItem)
+
+	// start a ticker which we'll use to output status and check for completion
+	ticker := time.Tick(250 * time.Millisecond)
+
+	// create first page's httpItem
+	homeitem, err := newHTTPItem(nil, homeurl)
+	if err != nil {
+		log.Fatal("unable to create httpItem for homeurl")
+	}
+
+	// set our number of outstanding pages (initially 1 to account for first page)
+	crawlingCount := 1
+
+	// start the home page crawl
+	crawled[homeitem.url.String()] = homeitem
+	go crawlItem(homeitem, rchan)
+
+	// wait for results
+	for {
+		select {
+		case r := <-rchan: // new results?
+			// add result to our results map
+			results[r.url.String()] = r
+
+			// decrease the outstanding page count by 1
+			crawlingCount--
+
+			// start crawly any new child pages we haven't yet crawled
+			for i, c := range r.children {
+				// see if we already have a result for this page
+				// if so, point to that result
+				if existing, ok := results[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
+				}
+
+				// see if we're already crawling this page (but maybe don't have results yet)
+				// if so, point to that item (we will have it as a result later!)
+				if existing, ok := crawled[c.url.String()]; ok {
+					r.children[i] = existing
+					continue
+				}
+
+				// haven't crawled this one yet, do so now
+				crawlingCount++
+				crawled[c.url.String()] = c
+				go crawlItem(c, rchan)
+			}
+
+		case <-ticker: // our regular ticker. for status output and checking for completion.
+			// output status to console
+			log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount)
+
+			// see if we're finished
+			if crawlingCount == 0 {
+				// finished! convert results map to a slice and return it
+				rslice := itemSlice{}
+				for _, v := range results {
+					rslice = append(rslice, v)
+				}
+				return rslice
+			}
+		}
+	}
+}
+
+// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it,
+// and filling out its structure as much as possible
+func crawlItem(item *httpItem, rchan chan<- *httpItem) {
+	// make sure this item is the same domain (i.e. URL "host part") as its referrer
+	if item.refurl != nil && item.url.Host != item.refurl.Host {
+		// skip URLs associated with other Hosts
+		item.linkType = tRemote
+		rchan <- item
+		return
+	}
+
+	// fetch page
+	text, err := fetchPage(item)
+	if err != nil {
+		rchan <- item
+		return
+	}
+
+	// parse links
+	title, links := parseLinks(text) // TODO remove error from parseLinks, not needed
+	if err != nil {
+		item.linkType = tBroken
+		rchan <- item
+		return
+	}
+	item.title = title
+
+	// walk links and add them as children to the current item
+	for _, l := range links {
+		newItem, err := newHTTPItem(item, l)
+		if err != nil {
+			continue // TODO bad item
+		}
+		item.children = append(item.children, newItem)
+	}
+
+	// send back our item struct now that it's all filled out
+	rchan <- item
+}
 
 func main() {
+	// crawl each URL on the command line
+	for _, u := range os.Args[1:] {
+		pages := doCrawl(u)
+		l := sitemapToLocations(pages)
+		j, _ := locationsToJSON(l)
+		fmt.Println(j)
+	}
 }

src/docrawler/docrawler_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -19,13 +18,13 @@ func TestMain(m *testing.M) {
 	// start our simple web server
 	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
 	if err != nil {
-		panic("unable to open port for http server") // TODO remove panic?
+		panic("unable to open port for http server")
 	}
 	http.Handle("/", http.FileServer(http.Dir("./testsite")))
 	go func() {
 		err = http.Serve(l, nil)
 		if err != nil {
-			panic("unable to start http server") // TODO remove panic?
+			panic("unable to start http server")
 		}
 	}()
 
@@ -43,8 +42,8 @@ func TestMain(m *testing.M) {
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error(err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal(err)
 	}
 }
 
@@ -52,21 +51,191 @@ func TestServerRunning(t *testing.T) {
 func TestBadRequest(t *testing.T) {
 	resp, err := http.Get(baseURL + "doesnotexist.html")
 	if resp.StatusCode != 404 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error("request for non existent file didn't fail properly")
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal("request for non existent file didn't fail properly")
 	}
 }
 
 // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
 func TestSimpleFetch(t *testing.T) {
-	body, err := fetch(baseURL + "fetch_test.html")
+	page, err := newHTTPItem(nil, baseURL+"fetch_test.html")
 	if err != nil {
-		t.Error(err)
+		t.Error("problem creating New httpItem struct")
+	}
+	body, err := fetchPage(page)
+	if err != nil {
+		t.Fatal(err)
 	}
 	desired := "<html><head></head><body></body></html>\n"
 	if body != desired {
-		fmt.Printf("   Got: '%v'\n", body)
-		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Logf("   Got: '%v'\n", body)
+		t.Logf("Wanted: '%v'\n", desired)
 		t.Error("fetch succeeded, but content mismatched")
 	}
 }
+
+// TestSimpleMap figures out the site map for the site in baseURL
+func TestSimpleMap(t *testing.T) {
+	pages := doCrawl(baseURL)
+
+	// because our crawl is non-deterministic, we have to do a complete
+	// cycle through every page, counting stuff, finding specific pages
+	// so that we can then verify certain aspects of each page
+	skipped, children := 0, 0
+	var homepage, aboutpage, skippage *httpItem
+	for _, p := range pages {
+		if p.title == "Home" {
+			homepage = p
+		} else if p.title == "About Test" {
+			aboutpage = p
+		} else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" {
+			skippage = p
+		}
+		if p.linkType == tRemote {
+			skipped++
+		}
+		children += len(p.children)
+	}
+
+	// check page counts
+	if len(pages) != 6 {
+		t.Logf("got %v, wanted %v\n", len(pages), 6)
+		t.Fatal("got wrong number of pages")
+	}
+	if children != 9 {
+		t.Logf("got %v, wanted %v\n", children, 8)
+		t.Fatal("got wrong number of total children")
+	}
+	if skipped != 1 {
+		t.Logf("got %v, wanted %v\n", skipped, 1)
+		t.Fatal("got wrong number of total skipped")
+	}
+
+	// check first page URL
+	if homepage.url.String() != baseURL {
+		t.Error("page URL is invalid")
+	}
+
+	// specific page number of children
+	if len(homepage.children) != 5 {
+		t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
+		t.Fatal("got wrong number of links")
+	}
+	if len(aboutpage.children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4)
+		t.Fatal("got wrong number of links")
+	}
+
+	// make sure remote pages were skipped
+	if homepage.linkType == tRemote {
+		t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String())
+		t.Error("child was skipped")
+	}
+	if skippage.linkType != tRemote {
+		t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String())
+		t.Error("child wasn't skipped")
+	}
+
+	// verify page content
+	// TODO find a better way to do this
+	//if homepage.children[0].url.String() != baseURL+"about.html" {
+	//t.Logf("   Got: %q\n", homepage.children[0].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"about.html")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.children[1].url.String() != baseURL+"assets/image.png" {
+	//t.Logf("   Got: %q\n", homepage.children[1].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.children[3].url.String() != baseURL+"scripts/blah.js" {
+	//t.Logf("   Got: %q\n", homepage.children[3].url.String())
+	//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+	//t.Error("link name is incorrect")
+	//}
+}
+
+// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
+func TestHeaderFetching(t *testing.T) {
+	basepage, err := newHTTPItem(nil, baseURL)
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err := newHTTPItem(basepage, "about.html")
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err = newHTTPItem(basepage, "assets/image.png")
+	if err != nil {
+		t.Error("problem creating New httpItem struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.linkType != tAsset {
+		t.Logf("got %q, wanted %q", page.linkType, tAsset)
+		t.Error("problem fetching filetype")
+	}
+}
+
+// TestJsonOutput gets a sitemap and then converts it to json
+func TestJsonOutput(t *testing.T) {
+	pages := doCrawl(baseURL)
+	l := sitemapToLocations(pages)
+	if len(l) != 2 {
+		t.Error("sitemapToLocations has the wrong number of locations")
+	}
+	if l[0].URL != baseURL {
+		t.Error("got an incorrect location")
+	}
+	if len(l[0].Links) != 1 {
+		t.Error("location 0 has wrong number of links")
+	}
+	if len(l[0].Assets) != 2 {
+		t.Logf("got %v", len(l[0].Assets))
+		t.Error("location 0 has wrong number of assets")
+	}
+	if len(l[0].Broken) != 1 {
+		t.Logf("got %v", len(l[0].Broken))
+		t.Error("location 0 has wrong number of broken urls")
+	}
+	if len(l[0].Remote) != 0 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 0 has wrong number of remote urls")
+	}
+	if l[1].URL != baseURL+"about.html" {
+		t.Error("got an incorrect location")
+	}
+	if len(l[1].Links) != 1 {
+		t.Error("location 1 has wrong number of links")
+	}
+	if len(l[1].Assets) != 2 {
+		t.Error("location 1 has wrong number of assets")
+	}
+	if len(l[1].Broken) != 0 {
+		t.Logf("got %v", len(l[1].Broken))
+		t.Error("location 1 has wrong number of broken urls")
+	}
+	if len(l[1].Remote) != 1 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 1 has wrong number of remote urls")
+	}
+
+	// this is a cheater test because the output is from a run of the code being
+	// test itself. but, it's been examined and I think it's right. and, there's not
+	// many other ways to test this without some significant pain
+	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]`
+	j, err := locationsToJSON(l)
+	if err != nil {
+		t.Error("locationsToJSON failed")
+	}
+	if j != cheaterTest {
+		t.Error("locationsToJSON text doesn't match")
+	}
+	t.Log(j)
+}

src/docrawler/fetch.go

@@ -1,18 +1,87 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 )
 
-// fetch takes a URL, GETs it, and returns the body as a string
-func fetch(url string) (string, error) {
-	// GET the url
-	resp, err := http.Get(url)
+// custom errors
+var (
+	errContentTypeNotFound = errors.New("no Content-Type header found")
+	errFetchError          = errors.New("couldn't fetch item")
+	errFileTypeUnknown     = errors.New("couldn't determine file type")
+)
+
+// fetchFiletype performs an http HEAD to get the media type, and sets it
+// directly in httpItem.mediaType
+func fetchFiletype(item *httpItem) error {
+	resp, err := http.Head(item.url.String())
+	if err != nil {
+		item.linkType = tBroken
+		return err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		item.linkType = tBroken
+		return errFetchError
+	}
+
+	// pull the content type out of the http header
+	contentType, ok := resp.Header["Content-Type"]
+	if !ok {
+		// no Content-Type header found
+		item.linkType = tBroken
+		return errContentTypeNotFound
+	}
+
+	// parse mime type
+	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		item.linkType = tBroken
+		return err
+	}
+
+	// success, set item type and return
+	if mediatype == "text/html" {
+		item.linkType = tHTMLPage
+	} else {
+		item.linkType = tAsset
+	}
+	return nil
+}
+
+// fetchPage takes a httpItem, GETs it, and returns the body as a string
+func fetchPage(item *httpItem) (string, error) {
+	// figure out the file type
+	err := fetchFiletype(item)
 	if err != nil {
 		return "", err
 	}
 
+	// make sure we got a file type!
+	if item.linkType == tUnknown {
+		return "", errFileTypeUnknown
+	}
+
+	// we only want to fetch html, so return if it's anything else
+	if item.linkType != tHTMLPage {
+		return "", nil // but this isn't an error!
+	}
+
+	// GET the url
+	resp, err := http.Get(item.url.String())
+	if err != nil {
+		return "", err
+	}
+
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return "", errFetchError
+	}
+
 	// read the body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {

src/docrawler/fetch_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedHeaderFetching(t *testing.T) {
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
+	}
+}
+
+// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedFetching(t *testing.T) {
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if _, err := fetchPage(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
+	}
+}

src/docrawler/item.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"net/url"
+)
+
+// itemSlice is a convenience type for a slice of items
+type itemSlice []*httpItem
+
+// itemMap convenience type for map of string's (URL) to items
+type itemMap map[string]*httpItem
+
+// itemType is an enum so we know how to classify each item
+type itemType int
+
+// enums for itemType
+const (
+	tUnknown itemType = iota
+	tHTMLPage
+	tAsset
+	tRemote
+	tBroken
+)
+
+// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+type httpItem struct {
+	url      *url.URL
+	refurl   *url.URL
+	title    string
+	linkType itemType
+	children itemSlice
+}
+
+// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{}
+func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.url.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// figure out referrer url
+	var rurl *url.URL
+	if referrer != nil {
+		rurl = referrer.url
+	}
+
+	// create struct and return
+	return &httpItem{url: u, refurl: rurl}, nil
+}

src/docrawler/item_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures
+func TestBadnewHTTPItem(t *testing.T) {
+	_, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	if err == nil {
+		t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error")
+	}
+}

src/docrawler/output.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc.
+type Location struct {
+	URL    string
+	Title  string
+	Links  []string
+	Assets []string
+	Broken []string
+	Remote []string
+}
+
+// implement Location slice sorting (by URL)
+type byURL []*Location
+
+// implement Sort interface on byURL (which is a []*Location)
+func (l byURL) Len() int           { return len(l) }
+func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL }
+func (l byURL) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
+
+// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling
+func sitemapToLocations(pages itemSlice) []*Location {
+	// build a map of our pages
+	pageMap := make(map[string]*httpItem)
+	for _, p := range pages {
+		pageMap[p.url.String()] = p
+	}
+
+	// build a slice of locations (one per page)
+	var locations []*Location
+	for _, p := range pages {
+		if p.linkType == tHTMLPage {
+			// create a location for this page
+			l := &Location{URL: p.url.String(), Title: p.title}
+
+			// add its children
+			for _, c := range p.children {
+				// look up this child's media type from the root list of pages
+				//mediaType := pageMap[c.url.String()].mediaType
+				if c.linkType == tRemote {
+					l.Remote = append(l.Remote, c.url.String())
+				} else if c.linkType == tHTMLPage {
+					l.Links = append(l.Links, c.url.String())
+				} else if c.linkType == tBroken {
+					l.Broken = append(l.Broken, c.url.String())
+				} else if c.linkType == tAsset {
+					l.Assets = append(l.Assets, c.url.String())
+				} else {
+					// unknown link here, which means it failed to crawl, let's call it "broken"
+					l.Broken = append(l.Broken, c.url.String())
+				}
+
+			}
+
+			// now uniq & sort the children slices
+			l.Remote = uniqStrings(l.Remote)
+			l.Links = uniqStrings(l.Links)
+			l.Broken = uniqStrings(l.Broken)
+			l.Assets = uniqStrings(l.Assets)
+			sort.Strings(l.Remote)
+			sort.Strings(l.Links)
+			sort.Strings(l.Broken)
+			sort.Strings(l.Assets)
+
+			// and add this location to our slice
+			locations = append(locations, l)
+		}
+	}
+
+	// sort the Locations themselves and return
+	sort.Sort(byURL(locations))
+	return locations
+}
+
+// locationsToJSON takes a *Location slice and marshals it into a JSON string
+func locationsToJSON(locations []*Location) (string, error) {
+	b, err := json.Marshal(locations)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

src/docrawler/parse.go

@@ -2,21 +2,21 @@ package main
 
 import (
 	"regexp"
-	"sort"
 	"strings"
 )
 
 // a regexp which captures src/href/xhref="..." from text
-var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
-var reTitle = regexp.MustCompile(`<\s*title\s*>([^<]*)`)
+var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html title and all links out of it,
-// and returns a sorted slice of the captures found
-func parse(s string) (string, []string, error) {
+// parseLinks takes a string and attempts to parse any html title and all links out of it,
+// and returns a slice of the captures found
+func parseLinks(s string) (string, []string) {
 	// find the title, or default to empty
 	title := ""
 	titleMatches := reTitle.FindStringSubmatch(s)
 	if titleMatches != nil {
+		// we found a title, replace our default title with the result
 		title = strings.TrimSpace(titleMatches[1])
 	}
 
@@ -25,6 +25,5 @@ func parse(s string) (string, []string, error) {
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	sort.Strings(results)
-	return title, results, nil
+	return title, results
 }

src/docrawler/parse_test.go

@@ -8,27 +8,28 @@ import (
 func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
-		<title> Test Page  </title>
+		<TITle> Test Page  </title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	title, matches, err := parse(doc)
-	if err != nil {
-		t.Error(err)
-	}
+	title, matches := parseLinks(doc)
 	if title != "Test Page" {
 		t.Error("got wrong title")
 	}
-	if len(matches) != 2 {
+	if len(matches) != 3 {
 		t.Error("invalid number of matches in parse")
 	}
-	if matches[0] != "/about.html" {
+	if matches[0] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
-	if matches[1] != "/assets/image.png" {
+	if matches[1] != "/about.html" {
+		t.Error("match text is invalid")
+	}
+	if matches[2] != "scripts/blah.js" {
 		t.Error("match text is invalid")
 	}
 }

src/docrawler/testsite/about.html

@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>About Test</title>
+		<script src="scripts/blah.js"/>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/">fooey!</a>
+		<a href="http://doesntexist23492387492837492374982734.com/">bad one</a>
+	</body>
+</html>

src/docrawler/testsite/index.html

@@ -1,8 +1,12 @@
 <html>
 	<head>
+		<title>Home</title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<a href="/about.html">duplicate!</a>
+		<script src="scripts/blah.js"/>
+		<a href="/zzzbroken.html">oops!</a>
 	</body>
 </html>

src/docrawler/testsite/scripts/blah.js

@@ -0,0 +1 @@
+

src/docrawler/url.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+// custom errors
+var (
+	errInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return errInvalidURL
+	}
+	return nil // success!
+}
+
+// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
+	if err != nil {
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
+
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
+}
+
+// stripAnchorFromURL returns a version of the URL with out the "Fragment" part,
+// which is anything after the '#' character, so:
+// http://a.com/blah.html#anchor becomes http://a.com/blah.html
+func stripAnchorFromURL(u *url.URL) string {
+	// copy the url locally
+	ucopy := *u
+
+	// blank out the anchor, return URI string
+	ucopy.Fragment = ""
+	return ucopy.String()
+}

src/docrawler/url_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"net/url"
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		t.Logf("   Got: %q\n", u.Host)
+		t.Logf("Wanted: %q\n", "somehost.com:8765")
+		t.Error("host extraction failed")
+	}
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := resolveURL(referrer, current)
+	if err != nil {
+		t.Error("resolveURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		t.Logf("   Got: %q\n", u)
+		t.Logf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
+}
+
+// TestBadResolve tries to resolve bogus URLs to be sure we're getting errors
+/* TODO not sure how to make url.Parse fail in these cases!!!
+func TestBadResolve(t* testing.T) {
+	u, err := resolveURL("http://a.com/", "h%20t\tp://doesnte%20xist23492387492837492374982734.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+	u, err = resolveURL("h%20t\tp://doesnte%20xist23492387492837492374982734.com/", "http://a.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+}
+*/
+
+// TestURLStripAnchor tests the stripAnchorFromURL() function
+func TestURLStripAnchor(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Fatal("couldn't parse a URL")
+	}
+	stripped := stripAnchorFromURL(u)
+	if stripped != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y" {
+		t.Error("stripping anchor #foo from URL failed")
+	}
+}

src/docrawler/util.go

@@ -0,0 +1,23 @@
+package main
+
+// uniqStrings takes a slice of strings and removes any duplicates
+// note: does not guarantee any order (or stability of order)
+func uniqStrings(strs []string) []string {
+	// handle special case of a nil parameter
+	if strs == nil {
+		return nil
+	}
+
+	// add each string to the slice as keys (used like a "set")
+	set := make(map[string]struct{})
+	for _, s := range strs {
+		set[s] = struct{}{}
+	}
+
+	// now copy all the keys over to a new slice and return it
+	result := []string{}
+	for key := range set {
+		result = append(result, key)
+	}
+	return result
+}

src/docrawler/util_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"sort"
+	"testing"
+)
+
+// TestUniqStrings passes a slice of strings to uniqStrings() to see if it removes
+// all duplicates
+func TestUniqStrings(t *testing.T) {
+	strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"}
+	uniq := uniqStrings(strs)
+	if len(uniq) != 5 {
+		t.Error("wrong number of strings after uniq'ing")
+	}
+	sort.Strings(uniq)
+	if uniq[0] != "a" || uniq[1] != "b" || uniq[2] != "c" || uniq[3] != "d" || uniq[4] != "e" {
+		t.Error("uniq values are in correct!")
+	}
+}
+
+// TestUniqStringsNilParm passes a nil slice to uniqStrings to be sure it returns nil
+func TestUniqStringsNilParm(t *testing.T) {
+	uniq := uniqStrings(nil)
+	if uniq != nil {
+		t.Error("uniqStrings should return a nil slice when passed a nil slice")
+	}
+}

util/testfilter.sh

@@ -2,6 +2,6 @@
 
 # colorize the output of "go test"
 
-sed ''/ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
+sed ''/^ok/s//$(printf "\033[32;1m&\033[0m")/''       | \
 	sed ''/PASS/s//$(printf "\033[32;1m&\033[0m")/''   | \
 	sed ''/FAIL/s//$(printf "\033[31;1m&\033[0m")/''