diff --git a/src/docrawler/url.go b/src/docrawler/url.go new file mode 100644 index 0000000..055886f --- /dev/null +++ b/src/docrawler/url.go @@ -0,0 +1,3 @@ +package main + +// scheme://[userinfo@]host/path[?query][#fragment] diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go new file mode 100644 index 0000000..ef08652 --- /dev/null +++ b/src/docrawler/url_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" +) + +// TestURLCleaner tests cleaning up (regularization) of URLs +func TestURLCleaner(t *testing.T) { + host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if host != "somehost.com" { + t.Error("host extraction failed") + } + if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + t.Error("url cleanup failed") + } +}