diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 2d088c4..99e4143 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,22 +1,51 @@ package main import ( + "errors" "net/url" "strings" ) -// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. -// returns the updated host and reassembled URL -func cleanURL(dirtyURL string) (string, string) { - // parse the URL into its parts - u, err := url.Parse(dirtyURL) +var ( + ErrInvalidURL = errors.New("this URL can't be parsed successfully") +) + +// cleanURL takes a URL and normalizes it by downcasing the host part +func cleanURL(u *url.URL) { + // convert host to all lower case, return updated url.URL + u.Host = strings.ToLower(u.Host) +} + +// checkURL looks to see if our URL meets our minimum requirements +func checkURL(u *url.URL) error { + // check for non-empty Host and Scheme + if len(u.Host) == 0 || len(u.Scheme) == 0 { + // something about this URL doesn't meet our spec + return ErrInvalidURL + } + return nil // success! +} + +// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL +// returns the final URL and any error +func getURL(referringURL string, currentURL string) (*url.URL, error) { + // begin by parsing both URLs + uCurrent, err := url.Parse(currentURL) if err != nil { - return "", dirtyURL // TODO what should we really return here? + return nil, err + } + uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse + if err != nil { + return nil, err } - // convert host to all lower case - u.Host = strings.ToLower(u.Host) + // normalize the URLs + cleanURL(uCurrent) + cleanURL(uReferrer) - // return the updated host and reassembled URL - return u.Host, u.String() + // try to resolve it with the referrer + uResolved := uReferrer.ResolveReference(uCurrent) + + // return the URL we ended up with, and the error from checkURL + return uResolved, checkURL(uResolved) } diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 61b7fd9..df6b45d 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -2,18 +2,53 @@ package main import ( "fmt" + "net/url" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { - host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com:8765" { - fmt.Printf(" Got: '%v'\n", host) - fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") + u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if err != nil { + t.Error("error parsing test URL") + } + cleanURL(u) + if u.Host != "somehost.com:8765" { + fmt.Printf(" Got: %q\n", u.Host) + fmt.Printf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } - if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { t.Error("url cleanup failed") } } + +// send in a bad URL and make sure we get nil back +func TestBadURL(t *testing.T) { + u, err := url.Parse("blah") + if err != nil { + t.Error("error parsing test URL") + } + if err := checkURL(u); err == nil { + t.Error("parsing bogus URL still returned some values") + } +} + +// some tests for the base url resolution (absolute vs. relative links) +func doResolveTest(t *testing.T, wanted, referrer, current string) { + u, err := getURL(referrer, current) + if err != nil { + t.Error("getURL failed (probably parsing)") + } + if u.String() != wanted { + fmt.Printf(" Got: %q\n", u) + fmt.Printf("Wanted: %q\n", wanted) + t.Error("url resolution with base URL failed") + } +} +func TestURLResolution(t *testing.T) { + doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html") + doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") + doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") + doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") +}