flow: Closed <feature> 'add_test_coverage'.

* also tweaked GOPATH as needed
* re-vendored with cover tool included
* ignore $/cover.out

.hgignore

@@ -1,2 +1,3 @@
 syntax: glob
-bin
+bin
+cover.out

Makefile

@@ -3,18 +3,19 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest run clean env vendor
+.PHONY: build fmt lint vet test autotest cover run clean env vendor
 
 # configuration
 APPNAME := docrawler
 VENDOR := .vendor
 PACKAGES :=                     \
 	golang.org/x/tools/cmd/vet    \
+	golang.org/x/tools/cmd/cover  \
 	github.com/golang/lint/golint
 
 # override GOPATH to be our vendor directory
-export GOPATH := ${PWD}/${VENDOR}
+export GOPATH := ${PWD}/${VENDOR}:${PWD}
-export PATH := ${PWD}/${VENDOR}/bin:${PATH}
+export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH}
 
 default: build
 
@@ -31,28 +32,31 @@ vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./util/testfilter.sh
+	go test -v ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \
-	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+		xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
+
+cover:
+	go test -coverprofile cover.out ./src/...  && go tool cover -html cover.out
 
 run: build
 	./bin/${APPNAME}
 
 clean:
-	rm -f ./bin/${APPNAME}
+	rm -f ./bin/${APPNAME} ./cover.out
 
 env:
 	go env
 
 # vendor
 # * wipes our vendor directory completely
-# * fetches all packages in ${PACKAGES}
+# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)
 # * removes source control dirs so packages are vendorable (commitable)
 vendor:
 	rm -dRf ./${VENDOR}/src                          && \
-	go get -u ${PACKAGES}                            && \
+	GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES}    && \
 	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
 	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
 	rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \

src/docrawler/parse.go

@@ -3,19 +3,29 @@ package main
 import (
 	"regexp"
 	"sort"
+	"strings"
 )
 
 // a regexp which captures src/href/xhref="..." from text
-var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice
+var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html links out of it,
+// parse takes a string and attempts to parse any html title and all links out of it,
 // and returns a sorted slice of the captures found
-func parse(s string) ([]string, error) {
+func parse(s string) (string, []string, error) {
-	// find all of the matches (submatch = include capture groups)
+	// find the title, or default to empty
+	title := ""
+	titleMatches := reTitle.FindStringSubmatch(s)
+	if titleMatches != nil {
+		// we found a title, replace our default title with the result
+		title = strings.TrimSpace(titleMatches[1])
+	}
+
+	// find all of the URL matches (submatch = include capture groups)
 	results := []string{}
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
 	sort.Strings(results)
-	return results, nil
+	return title, results, nil
 }

src/docrawler/parse_test.go

@@ -4,20 +4,24 @@ import (
 	"testing"
 )
 
-// TestSimpleParse verifies that we can extract URLs (etc.) from a known document
+// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document
 func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
+		<TITle> Test Page  </title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
 		<a href="/about.html">
 	</body>
 </html>`
-	matches, err := parse(doc)
+	title, matches, err := parse(doc)
 	if err != nil {
 		t.Error(err)
 	}
+	if title != "Test Page" {
+		t.Error("got wrong title")
+	}
 	if len(matches) != 2 {
 		t.Error("invalid number of matches in parse")
 	}

src/docrawler/url.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+var (
+	ErrInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return ErrInvalidURL
+	}
+	return nil // success!
+}
+
+// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
+	if err != nil {
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
+
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
+}

src/docrawler/url_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		fmt.Printf("   Got: %q\n", u.Host)
+		fmt.Printf("Wanted: %q\n", "somehost.com:8765")
+		t.Error("host extraction failed")
+	}
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := resolveURL(referrer, current)
+	if err != nil {
+		t.Error("resolveURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		fmt.Printf("   Got: %q\n", u)
+		fmt.Printf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+}