flow: Closed <feature> 'url_parse_and_clean'.

* test cases (relative and absolute) added
* decent cleanup (using url.URL throughout instead of strings)

Makefile

@@ -31,7 +31,7 @@ vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./util/testfilter.sh
+	go test -v ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \

src/docrawler/parse.go

@@ -3,19 +3,29 @@ package main
 import (
 	"regexp"
 	"sort"
+	"strings"
 )
 
 // a regexp which captures src/href/xhref="..." from text
-var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice
+var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html links out of it,
+// parse takes a string and attempts to parse any html title and all links out of it,
 // and returns a sorted slice of the captures found
-func parse(s string) ([]string, error) {
-	// find all of the matches (submatch = include capture groups)
+func parse(s string) (string, []string, error) {
+	// find the title, or default to empty
+	title := ""
+	titleMatches := reTitle.FindStringSubmatch(s)
+	if titleMatches != nil {
+		// we found a title, replace our default title with the result
+		title = strings.TrimSpace(titleMatches[1])
+	}
+
+	// find all of the URL matches (submatch = include capture groups)
 	results := []string{}
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
 	sort.Strings(results)
-	return results, nil
+	return title, results, nil
 }

src/docrawler/parse_test.go

@@ -4,20 +4,24 @@ import (
 	"testing"
 )
 
-// TestSimpleParse verifies that we can extract URLs (etc.) from a known document
+// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document
 func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
+		<TITle> Test Page  </title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
 		<a href="/about.html">
 	</body>
 </html>`
-	matches, err := parse(doc)
+	title, matches, err := parse(doc)
 	if err != nil {
 		t.Error(err)
 	}
+	if title != "Test Page" {
+		t.Error("got wrong title")
+	}
 	if len(matches) != 2 {
 		t.Error("invalid number of matches in parse")
 	}

src/docrawler/url.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+var (
+	ErrInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return ErrInvalidURL
+	}
+	return nil // success!
+}
+
+// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
+	if err != nil {
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
+	}
+
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
+
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
+}

src/docrawler/url_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		fmt.Printf("   Got: %q\n", u.Host)
+		fmt.Printf("Wanted: %q\n", "somehost.com:8765")
+		t.Error("host extraction failed")
+	}
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := resolveURL(referrer, current)
+	if err != nil {
+		t.Error("resolveURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		fmt.Printf("   Got: %q\n", u)
+		fmt.Printf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+}