From 5cad35e382437e2aaf520f8fa67a9d8194e44fdb Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:14:29 -0800 Subject: [PATCH 1/6] flow: Created branch 'feature/url_parse_and_clean'. From 42799df06eb571dd389edc51045f30c008484185 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:20:00 -0800 Subject: [PATCH 2/6] add initial failing test for cleanURL --- src/docrawler/url.go | 3 +++ src/docrawler/url_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/docrawler/url.go create mode 100644 src/docrawler/url_test.go diff --git a/src/docrawler/url.go b/src/docrawler/url.go new file mode 100644 index 0000000..055886f --- /dev/null +++ b/src/docrawler/url.go @@ -0,0 +1,3 @@ +package main + +// scheme://[userinfo@]host/path[?query][#fragment] diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go new file mode 100644 index 0000000..ef08652 --- /dev/null +++ b/src/docrawler/url_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" +) + +// TestURLCleaner tests cleaning up (regularization) of URLs +func TestURLCleaner(t *testing.T) { + host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if host != "somehost.com" { + t.Error("host extraction failed") + } + if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + t.Error("url cleanup failed") + } +} From 24f557d92768855059693ee662e53d1f17db727e Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:29:41 -0800 Subject: [PATCH 3/6] basic url cleaning test passes --- src/docrawler/url.go | 21 ++++++++++++++++++++- src/docrawler/url_test.go | 5 ++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 055886f..2d088c4 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,3 +1,22 @@ package main -// scheme://[userinfo@]host/path[?query][#fragment] +import ( + "net/url" + "strings" +) + +// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. +// returns the updated host and reassembled URL +func cleanURL(dirtyURL string) (string, string) { + // parse the URL into its parts + u, err := url.Parse(dirtyURL) + if err != nil { + return "", dirtyURL // TODO what should we really return here? + } + + // convert host to all lower case + u.Host = strings.ToLower(u.Host) + + // return the updated host and reassembled URL + return u.Host, u.String() +} diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index ef08652..61b7fd9 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,13 +1,16 @@ package main import ( + "fmt" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com" { + if host != "somehost.com:8765" { + fmt.Printf(" Got: '%v'\n", host) + fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") t.Error("host extraction failed") } if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { From b398e2ddf44dfc4481eaad6d2b63bcb66b787120 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:00:06 -0800 Subject: [PATCH 4/6] url resolution (with baseref) now functional * test cases (relative and absolute) added * decent cleanup (using url.URL throughout instead of strings) --- src/docrawler/url.go | 49 +++++++++++++++++++++++++++++++-------- src/docrawler/url_test.go | 45 +++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 2d088c4..99e4143 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,22 +1,51 @@ package main import ( + "errors" "net/url" "strings" ) -// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. -// returns the updated host and reassembled URL -func cleanURL(dirtyURL string) (string, string) { - // parse the URL into its parts - u, err := url.Parse(dirtyURL) +var ( + ErrInvalidURL = errors.New("this URL can't be parsed successfully") +) + +// cleanURL takes a URL and normalizes it by downcasing the host part +func cleanURL(u *url.URL) { + // convert host to all lower case, return updated url.URL + u.Host = strings.ToLower(u.Host) +} + +// checkURL looks to see if our URL meets our minimum requirements +func checkURL(u *url.URL) error { + // check for non-empty Host and Scheme + if len(u.Host) == 0 || len(u.Scheme) == 0 { + // something about this URL doesn't meet our spec + return ErrInvalidURL + } + return nil // success! +} + +// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL +// returns the final URL and any error +func getURL(referringURL string, currentURL string) (*url.URL, error) { + // begin by parsing both URLs + uCurrent, err := url.Parse(currentURL) if err != nil { - return "", dirtyURL // TODO what should we really return here? + return nil, err + } + uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse + if err != nil { + return nil, err } - // convert host to all lower case - u.Host = strings.ToLower(u.Host) + // normalize the URLs + cleanURL(uCurrent) + cleanURL(uReferrer) - // return the updated host and reassembled URL - return u.Host, u.String() + // try to resolve it with the referrer + uResolved := uReferrer.ResolveReference(uCurrent) + + // return the URL we ended up with, and the error from checkURL + return uResolved, checkURL(uResolved) } diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 61b7fd9..df6b45d 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -2,18 +2,53 @@ package main import ( "fmt" + "net/url" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { - host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com:8765" { - fmt.Printf(" Got: '%v'\n", host) - fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") + u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if err != nil { + t.Error("error parsing test URL") + } + cleanURL(u) + if u.Host != "somehost.com:8765" { + fmt.Printf(" Got: %q\n", u.Host) + fmt.Printf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } - if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { t.Error("url cleanup failed") } } + +// send in a bad URL and make sure we get nil back +func TestBadURL(t *testing.T) { + u, err := url.Parse("blah") + if err != nil { + t.Error("error parsing test URL") + } + if err := checkURL(u); err == nil { + t.Error("parsing bogus URL still returned some values") + } +} + +// some tests for the base url resolution (absolute vs. relative links) +func doResolveTest(t *testing.T, wanted, referrer, current string) { + u, err := getURL(referrer, current) + if err != nil { + t.Error("getURL failed (probably parsing)") + } + if u.String() != wanted { + fmt.Printf(" Got: %q\n", u) + fmt.Printf("Wanted: %q\n", wanted) + t.Error("url resolution with base URL failed") + } +} +func TestURLResolution(t *testing.T) { + doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html") + doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") + doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") + doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") +} From 65b119ace66cf44e8bba704c86311aba519deb67 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:02:20 -0800 Subject: [PATCH 5/6] rename getURL() to resolveURL() because "get" may imply actually GETting it --- src/docrawler/url.go | 4 ++-- src/docrawler/url_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 99e4143..0e90021 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -26,9 +26,9 @@ func checkURL(u *url.URL) error { return nil // success! } -// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL +// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL // returns the final URL and any error -func getURL(referringURL string, currentURL string) (*url.URL, error) { +func resolveURL(referringURL string, currentURL string) (*url.URL, error) { // begin by parsing both URLs uCurrent, err := url.Parse(currentURL) if err != nil { diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index df6b45d..5381b64 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -36,9 +36,9 @@ func TestBadURL(t *testing.T) { // some tests for the base url resolution (absolute vs. relative links) func doResolveTest(t *testing.T, wanted, referrer, current string) { - u, err := getURL(referrer, current) + u, err := resolveURL(referrer, current) if err != nil { - t.Error("getURL failed (probably parsing)") + t.Error("resolveURL failed (probably parsing)") } if u.String() != wanted { fmt.Printf(" Got: %q\n", u) From afa5cfcc89383924ab5a8293f32049cf1a1aa3e2 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:02:25 -0800 Subject: [PATCH 6/6] flow: Closed 'url_parse_and_clean'.