url resolution (with baseref) now functional

* test cases (relative and absolute) added
* decent cleanup (using url.URL throughout instead of strings)

src/docrawler/url.go

@@ -1,22 +1,51 @@
 package main
 
 import (
+	"errors"
 	"net/url"
 	"strings"
 )
 
-// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it.
-// returns the updated host and reassembled URL
-func cleanURL(dirtyURL string) (string, string) {
-	// parse the URL into its parts
-	u, err := url.Parse(dirtyURL)
+var (
+	ErrInvalidURL = errors.New("this URL can't be parsed successfully")
+)
+
+// cleanURL takes a URL and normalizes it by downcasing the host part
+func cleanURL(u *url.URL) {
+	// convert host to all lower case, return updated url.URL
+	u.Host = strings.ToLower(u.Host)
+}
+
+// checkURL looks to see if our URL meets our minimum requirements
+func checkURL(u *url.URL) error {
+	// check for non-empty Host and Scheme
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
+		// something about this URL doesn't meet our spec
+		return ErrInvalidURL
+	}
+	return nil // success!
+}
+
+// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL
+// returns the final URL and any error
+func getURL(referringURL string, currentURL string) (*url.URL, error) {
+	// begin by parsing both URLs
+	uCurrent, err := url.Parse(currentURL)
 	if err != nil {
-		return "", dirtyURL // TODO what should we really return here?
+		return nil, err
+	}
+	uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse
+	if err != nil {
+		return nil, err
 	}
 
-	// convert host to all lower case
-	u.Host = strings.ToLower(u.Host) 
+	// normalize the URLs
+	cleanURL(uCurrent)
+	cleanURL(uReferrer)
 
-	// return the updated host and reassembled URL
-	return u.Host, u.String()
+	// try to resolve it with the referrer
+	uResolved := uReferrer.ResolveReference(uCurrent)
+
+	// return the URL we ended up with, and the error from checkURL
+	return uResolved, checkURL(uResolved)
 }

src/docrawler/url_test.go

@@ -2,18 +2,53 @@ package main
 
 import (
 	"fmt"
+	"net/url"
 	"testing"
 )
 
 // TestURLCleaner tests cleaning up (regularization) of URLs
 func TestURLCleaner(t *testing.T) {
-	host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
-	if host != "somehost.com:8765" {
-		fmt.Printf("   Got: '%v'\n", host)
-		fmt.Printf("Wanted: '%v'\n", "somehost.com:8765")
+	u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	cleanURL(u)
+	if u.Host != "somehost.com:8765" {
+		fmt.Printf("   Got: %q\n", u.Host)
+		fmt.Printf("Wanted: %q\n", "somehost.com:8765")
 		t.Error("host extraction failed")
 	}
-	if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
 		t.Error("url cleanup failed")
 	}
 }
+
+// send in a bad URL and make sure we get nil back
+func TestBadURL(t *testing.T) {
+	u, err := url.Parse("blah")
+	if err != nil {
+		t.Error("error parsing test URL")
+	}
+	if err := checkURL(u); err == nil {
+		t.Error("parsing bogus URL still returned some values")
+	}
+}
+
+// some tests for the base url resolution (absolute vs. relative links)
+func doResolveTest(t *testing.T, wanted, referrer, current string) {
+	u, err := getURL(referrer, current)
+	if err != nil {
+		t.Error("getURL failed (probably parsing)")
+	}
+	if u.String() != wanted {
+		fmt.Printf("   Got: %q\n", u)
+		fmt.Printf("Wanted: %q\n", wanted)
+		t.Error("url resolution with base URL failed")
+	}
+}
+func TestURLResolution(t *testing.T) {
+	doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html")
+	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
+	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
+	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+}