From d716fbc6f896f5fd0a633d524e9ca1bcc851f664 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sat, 14 Feb 2015 23:50:10 -0800 Subject: [PATCH 01/46] flow initialization: Created trunk: develop From 38224a89874e568197f95c198bd8e3f1a7dc40f2 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 00:40:36 -0800 Subject: [PATCH 02/46] misc. minor cleanup * moved $/testfilter.sh to $/util/testfilter.sh (updated Makefile accordingly) * updated idea in Design.md * renamed TestParse/TestFetch to TestSimpleParse/TestSimpleFetch.. more tests coming * parse now sorts its results. test updated. * moved regexp compilation to init instead of every time parse() is called * misc comments and cleanup --- Makefile | 4 ++-- docs/Design.md | 1 + src/docrawler/docrawler_test.go | 12 ++++++------ src/docrawler/fetch.go | 3 +++ src/docrawler/parse.go | 12 ++++++++---- testfilter.sh => util/testfilter.sh | 0 6 files changed, 20 insertions(+), 12 deletions(-) rename testfilter.sh => util/testfilter.sh (100%) diff --git a/Makefile b/Makefile index 9c4855c..e4cdaac 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,11 @@ vet: go vet ./src/... test: - go test ./src/... | ./testfilter.sh + go test ./src/... | ./util/testfilter.sh autotest: fswatch -or ./src | \ - xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./testfilter.sh' + xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh' run: build ./bin/${APPNAME} diff --git a/docs/Design.md b/docs/Design.md index 183f389..d0e11ee 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -39,6 +39,7 @@ These are features I'd like to include. * use channels to form a pipeline * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines) +* pipeline may allows us to scale concurrency at the bottlenecks * could use Interfaces for things like Parser, Getter, but I think best avoided until needed * launch goroutine per fetch * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index a2c50c7..10dae64 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -57,8 +57,8 @@ func TestBadRequest(t *testing.T) { } } -// TestFetch makes sure we can fetch a file and we get exactly what we expect -func TestFetch(t *testing.T) { +// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect +func TestSimpleFetch(t *testing.T) { body, err := fetch(baseURL + "fetch_test.html") if err != nil { t.Error(err) @@ -71,8 +71,8 @@ func TestFetch(t *testing.T) { } } -// TestParse verifies that we can extract URLs (etc.) from a known document -func TestParse(t *testing.T) { +// TestSimpleParse verifies that we can extract URLs (etc.) from a known document +func TestSimpleParse(t *testing.T) { doc := ` @@ -88,10 +88,10 @@ func TestParse(t *testing.T) { if len(matches) != 2 { t.Error("invalid number of matches in parse") } - if matches[0] != "/assets/image.png" { + if matches[0] != "/about.html" { t.Error("match text is invalid") } - if matches[1] != "/about.html" { + if matches[1] != "/assets/image.png" { t.Error("match text is invalid") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index bdf1836..4142836 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -5,6 +5,7 @@ import ( "net/http" ) +// fetch takes a URL, GETs it, and returns the body as a string func fetch(url string) (string, error) { // GET the url resp, err := http.Get(url) @@ -17,6 +18,8 @@ func fetch(url string) (string, error) { if err != nil { return "", err } + + // success! close the body, return it as a string resp.Body.Close() return string(body), nil } diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 483dbdb..30e678a 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -2,16 +2,20 @@ package main import ( "regexp" + "sort" ) -func parse(s string) ([]string, error) { - // hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.) - re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) +// a regexp which captures src/href/xhref="..." from text +var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice +// parse takes a string and attempts to parse any html links out of it, +// and returns a sorted slice of the captures found +func parse(s string) ([]string, error) { // find all of the matches (submatch = include capture groups) results := []string{} - for _, match := range re.FindAllStringSubmatch(s, -1) { + for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } + sort.Strings(results) return results, nil } diff --git a/testfilter.sh b/util/testfilter.sh similarity index 100% rename from testfilter.sh rename to util/testfilter.sh From 249aca229b6196e2cafce24fcb29c5f55f2dc364 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 00:43:50 -0800 Subject: [PATCH 03/46] flow: Created branch 'feature/create_simple_sitemap'. From 5ef46be8c4d3d44181971dced04d90d4a5832c8f Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 00:58:01 -0800 Subject: [PATCH 04/46] add (presently failing) tests for discovering a simple site map --- src/docrawler/docrawler.go | 13 +++++++++++++ src/docrawler/docrawler_test.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index c8f4ae0..1134f4a 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,5 +2,18 @@ package main import () +// Page is a struct which defines a single page, which URLs (links and assets) it contains +type Page struct { + URL string + Assets []string + Links []string +} + +// docrawl begins crawling the site at "url" +func docrawl(url string) []Page { + var result []Page + return result +} + func main() { } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 10dae64..4b491fb 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -44,7 +44,7 @@ func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error(err) + t.Fatal(err) } } @@ -53,7 +53,7 @@ func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error("request for non existent file didn't fail properly") + t.Fatal("request for non existent file didn't fail properly") } } @@ -61,7 +61,7 @@ func TestBadRequest(t *testing.T) { func TestSimpleFetch(t *testing.T) { body, err := fetch(baseURL + "fetch_test.html") if err != nil { - t.Error(err) + t.Fatal(err) } desired := "\n" if body != desired { @@ -86,7 +86,7 @@ func TestSimpleParse(t *testing.T) { t.Error(err) } if len(matches) != 2 { - t.Error("invalid number of matches in parse") + t.Fatal("invalid number of matches in parse") } if matches[0] != "/about.html" { t.Error("match text is invalid") @@ -95,3 +95,26 @@ func TestSimpleParse(t *testing.T) { t.Error("match text is invalid") } } + +// TestSimpleMap figures out the site map for the site in baseURL +func TestSimpleMap(t *testing.T) { + pages := docrawl(baseURL) + if len(pages) != 1 { + t.Fatal("got wrong number of pages") + } + if pages[0].URL != baseURL { + t.Error("page URL is invalid") + } + if len(pages[0].Assets) != 1 { + t.Fatal("got wrong number of assets") + } + if pages[0].Assets[0] != baseURL+"/assets/image.png" { + t.Error("asset name is incorrect") + } + if len(pages[0].Links) != 1 { + t.Fatal("got wrong number of links") + } + if pages[0].Links[0] != baseURL+"/about.html" { + t.Error("link name is incorrect") + } +} From 0cd3cb510af3a9e22ec33dad1fe4920b1080031e Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 07:40:57 -0800 Subject: [PATCH 05/46] flow: Created branch 'feature/split_tests'. From 042812f38e0730c2abff0a1db50eebb0503e2022 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 07:45:06 -0800 Subject: [PATCH 06/46] move simple parse test to parse_test.go which apply only to the parser. tests in dowcrawler_test.go remain integration/functional tests. --- src/docrawler/docrawler_test.go | 25 ------------------------- src/docrawler/parse_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 src/docrawler/parse_test.go diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 10dae64..59d7aaa 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -70,28 +70,3 @@ func TestSimpleFetch(t *testing.T) { t.Error("fetch succeeded, but content mismatched") } } - -// TestSimpleParse verifies that we can extract URLs (etc.) from a known document -func TestSimpleParse(t *testing.T) { - doc := ` - - - - - - -` - matches, err := parse(doc) - if err != nil { - t.Error(err) - } - if len(matches) != 2 { - t.Error("invalid number of matches in parse") - } - if matches[0] != "/about.html" { - t.Error("match text is invalid") - } - if matches[1] != "/assets/image.png" { - t.Error("match text is invalid") - } -} diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go new file mode 100644 index 0000000..738f2c4 --- /dev/null +++ b/src/docrawler/parse_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" +) + +// TestSimpleParse verifies that we can extract URLs (etc.) from a known document +func TestSimpleParse(t *testing.T) { + doc := ` + + + + + + +` + matches, err := parse(doc) + if err != nil { + t.Error(err) + } + if len(matches) != 2 { + t.Error("invalid number of matches in parse") + } + if matches[0] != "/about.html" { + t.Error("match text is invalid") + } + if matches[1] != "/assets/image.png" { + t.Error("match text is invalid") + } +} From 57b7130ac776734acbe7477e855169b74fb5c905 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 07:45:16 -0800 Subject: [PATCH 07/46] flow: Closed 'split_tests'. From 7501abf12180a34b6f999fad73188704effa9786 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 07:48:33 -0800 Subject: [PATCH 08/46] flow: Created branch 'feature/extract_titles_from_pages'. From b01fcb2132fa85537dd5d99883226ce35174e48f Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:03:19 -0800 Subject: [PATCH 09/46] add parsing of ... to parse() --- src/docrawler/parse.go | 19 ++++++++++++++----- src/docrawler/parse_test.go | 8 ++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 30e678a..e03a667 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -3,19 +3,28 @@ package main import ( "regexp" "sort" + "strings" ) // a regexp which captures src/href/xhref="..." from text -var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice +var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant +var reTitle = regexp.MustCompile(`<\s*title\s*>([^<]*)`) -// parse takes a string and attempts to parse any html links out of it, +// parse takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found -func parse(s string) ([]string, error) { - // find all of the matches (submatch = include capture groups) +func parse(s string) (string, []string, error) { + // find the title, or default to empty + title := "" + titleMatches := reTitle.FindStringSubmatch(s) + if titleMatches != nil { + title = strings.TrimSpace(titleMatches[1]) + } + + // find all of the URL matches (submatch = include capture groups) results := []string{} for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } sort.Strings(results) - return results, nil + return title, results, nil } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index 738f2c4..eae5883 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -4,20 +4,24 @@ import ( "testing" ) -// TestSimpleParse verifies that we can extract URLs (etc.) from a known document +// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document func TestSimpleParse(t *testing.T) { doc := ` + Test Page ` - matches, err := parse(doc) + title, matches, err := parse(doc) if err != nil { t.Error(err) } + if title != "Test Page" { + t.Error("got wrong title") + } if len(matches) != 2 { t.Error("invalid number of matches in parse") } From b70a89815ec2ec033c14495ad5e110ff307ffa1f Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:03:28 -0800 Subject: [PATCH 10/46] flow: Closed 'extract_titles_from_pages'. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e4cdaac..aefe125 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ vet: go vet ./src/... test: - go test ./src/... | ./util/testfilter.sh + go test -v ./src/... | ./util/testfilter.sh autotest: fswatch -or ./src | \ From 0e58ea12822623f12b0c3d93e7dd4876d8fa5822 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:07:42 -0800 Subject: [PATCH 11/46] bugfix: enable case insensitivity on REs for urls and title --- src/docrawler/parse.go | 5 +++-- src/docrawler/parse_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index e03a667..13a1010 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -7,8 +7,8 @@ import ( ) // a regexp which captures src/href/xhref="..." from text -var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant -var reTitle = regexp.MustCompile(`<\s*title\s*>([^<]*)`) +var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant +var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) // parse takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found @@ -17,6 +17,7 @@ func parse(s string) (string, []string, error) { title := "" titleMatches := reTitle.FindStringSubmatch(s) if titleMatches != nil { + // we found a title, replace our default title with the result title = strings.TrimSpace(titleMatches[1]) } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index eae5883..9089d55 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -8,7 +8,7 @@ import ( func TestSimpleParse(t *testing.T) { doc := ` - Test Page + Test Page From 5cad35e382437e2aaf520f8fa67a9d8194e44fdb Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:14:29 -0800 Subject: [PATCH 12/46] flow: Created branch 'feature/url_parse_and_clean'. From 42799df06eb571dd389edc51045f30c008484185 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:20:00 -0800 Subject: [PATCH 13/46] add initial failing test for cleanURL --- src/docrawler/url.go | 3 +++ src/docrawler/url_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/docrawler/url.go create mode 100644 src/docrawler/url_test.go diff --git a/src/docrawler/url.go b/src/docrawler/url.go new file mode 100644 index 0000000..055886f --- /dev/null +++ b/src/docrawler/url.go @@ -0,0 +1,3 @@ +package main + +// scheme://[userinfo@]host/path[?query][#fragment] diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go new file mode 100644 index 0000000..ef08652 --- /dev/null +++ b/src/docrawler/url_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" +) + +// TestURLCleaner tests cleaning up (regularization) of URLs +func TestURLCleaner(t *testing.T) { + host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if host != "somehost.com" { + t.Error("host extraction failed") + } + if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + t.Error("url cleanup failed") + } +} From 24f557d92768855059693ee662e53d1f17db727e Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 08:29:41 -0800 Subject: [PATCH 14/46] basic url cleaning test passes --- src/docrawler/url.go | 21 ++++++++++++++++++++- src/docrawler/url_test.go | 5 ++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 055886f..2d088c4 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,3 +1,22 @@ package main -// scheme://[userinfo@]host/path[?query][#fragment] +import ( + "net/url" + "strings" +) + +// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. +// returns the updated host and reassembled URL +func cleanURL(dirtyURL string) (string, string) { + // parse the URL into its parts + u, err := url.Parse(dirtyURL) + if err != nil { + return "", dirtyURL // TODO what should we really return here? + } + + // convert host to all lower case + u.Host = strings.ToLower(u.Host) + + // return the updated host and reassembled URL + return u.Host, u.String() +} diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index ef08652..61b7fd9 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,13 +1,16 @@ package main import ( + "fmt" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com" { + if host != "somehost.com:8765" { + fmt.Printf(" Got: '%v'\n", host) + fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") t.Error("host extraction failed") } if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { From b398e2ddf44dfc4481eaad6d2b63bcb66b787120 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:00:06 -0800 Subject: [PATCH 15/46] url resolution (with baseref) now functional * test cases (relative and absolute) added * decent cleanup (using url.URL throughout instead of strings) --- src/docrawler/url.go | 49 +++++++++++++++++++++++++++++++-------- src/docrawler/url_test.go | 45 +++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 2d088c4..99e4143 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,22 +1,51 @@ package main import ( + "errors" "net/url" "strings" ) -// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. -// returns the updated host and reassembled URL -func cleanURL(dirtyURL string) (string, string) { - // parse the URL into its parts - u, err := url.Parse(dirtyURL) +var ( + ErrInvalidURL = errors.New("this URL can't be parsed successfully") +) + +// cleanURL takes a URL and normalizes it by downcasing the host part +func cleanURL(u *url.URL) { + // convert host to all lower case, return updated url.URL + u.Host = strings.ToLower(u.Host) +} + +// checkURL looks to see if our URL meets our minimum requirements +func checkURL(u *url.URL) error { + // check for non-empty Host and Scheme + if len(u.Host) == 0 || len(u.Scheme) == 0 { + // something about this URL doesn't meet our spec + return ErrInvalidURL + } + return nil // success! +} + +// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL +// returns the final URL and any error +func getURL(referringURL string, currentURL string) (*url.URL, error) { + // begin by parsing both URLs + uCurrent, err := url.Parse(currentURL) if err != nil { - return "", dirtyURL // TODO what should we really return here? + return nil, err + } + uReferrer, err := url.Parse(referringURL) // many times this will be an extraneous parse + if err != nil { + return nil, err } - // convert host to all lower case - u.Host = strings.ToLower(u.Host) + // normalize the URLs + cleanURL(uCurrent) + cleanURL(uReferrer) - // return the updated host and reassembled URL - return u.Host, u.String() + // try to resolve it with the referrer + uResolved := uReferrer.ResolveReference(uCurrent) + + // return the URL we ended up with, and the error from checkURL + return uResolved, checkURL(uResolved) } diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 61b7fd9..df6b45d 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -2,18 +2,53 @@ package main import ( "fmt" + "net/url" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { - host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com:8765" { - fmt.Printf(" Got: '%v'\n", host) - fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") + u, err := url.Parse("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") + if err != nil { + t.Error("error parsing test URL") + } + cleanURL(u) + if u.Host != "somehost.com:8765" { + fmt.Printf(" Got: %q\n", u.Host) + fmt.Printf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } - if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { + if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { t.Error("url cleanup failed") } } + +// send in a bad URL and make sure we get nil back +func TestBadURL(t *testing.T) { + u, err := url.Parse("blah") + if err != nil { + t.Error("error parsing test URL") + } + if err := checkURL(u); err == nil { + t.Error("parsing bogus URL still returned some values") + } +} + +// some tests for the base url resolution (absolute vs. relative links) +func doResolveTest(t *testing.T, wanted, referrer, current string) { + u, err := getURL(referrer, current) + if err != nil { + t.Error("getURL failed (probably parsing)") + } + if u.String() != wanted { + fmt.Printf(" Got: %q\n", u) + fmt.Printf("Wanted: %q\n", wanted) + t.Error("url resolution with base URL failed") + } +} +func TestURLResolution(t *testing.T) { + doResolveTest(t, "http://base.com/about.html", "http://BaSe.CoM/index.html", "about.html") + doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") + doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") + doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") +} From 65b119ace66cf44e8bba704c86311aba519deb67 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:02:20 -0800 Subject: [PATCH 16/46] rename getURL() to resolveURL() because "get" may imply actually GETting it --- src/docrawler/url.go | 4 ++-- src/docrawler/url_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 99e4143..0e90021 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -26,9 +26,9 @@ func checkURL(u *url.URL) error { return nil // success! } -// getURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL +// resolveURL takes a URL and its referring URL and tries to parse it into an acceptable complete URL // returns the final URL and any error -func getURL(referringURL string, currentURL string) (*url.URL, error) { +func resolveURL(referringURL string, currentURL string) (*url.URL, error) { // begin by parsing both URLs uCurrent, err := url.Parse(currentURL) if err != nil { diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index df6b45d..5381b64 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -36,9 +36,9 @@ func TestBadURL(t *testing.T) { // some tests for the base url resolution (absolute vs. relative links) func doResolveTest(t *testing.T, wanted, referrer, current string) { - u, err := getURL(referrer, current) + u, err := resolveURL(referrer, current) if err != nil { - t.Error("getURL failed (probably parsing)") + t.Error("resolveURL failed (probably parsing)") } if u.String() != wanted { fmt.Printf(" Got: %q\n", u) From afa5cfcc89383924ab5a8293f32049cf1a1aa3e2 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 10:02:25 -0800 Subject: [PATCH 17/46] flow: Closed 'url_parse_and_clean'. From 0456723f0ba510030c2e2f6ceee6a9d5bfacbe68 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 12:12:30 -0800 Subject: [PATCH 18/46] minimally functional crawler - crawls single page, finds links, stores in data structure * refined Page struct. now need to refactor everything else to use this * wrote docrawl() function. NOT intended to be anything like the final design. just sketching it out to get a better idea of needs, design, etc. * replaced fmt.Print* with t.Log* in tests (oops!) * added a header fetcher function to get the file type (since we only want to parse html (with tests) * added small test for the URL resolver where the base URL is empty * added placeholder files in testsite/ for about.html and image.png --- src/docrawler/docrawler.go | 64 ++++++++++++++++++++++++++---- src/docrawler/docrawler_test.go | 45 ++++++++++++++------- src/docrawler/fetch.go | 31 +++++++++++++++ src/docrawler/testsite/about.html | 0 src/docrawler/testsite/image.png | Bin 0 -> 938 bytes src/docrawler/url_test.go | 10 ++--- 6 files changed, 123 insertions(+), 27 deletions(-) create mode 100644 src/docrawler/testsite/about.html create mode 100644 src/docrawler/testsite/image.png diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 1134f4a..508fe56 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,18 +1,66 @@ package main -import () +import ( + "net/url" +) -// Page is a struct which defines a single page, which URLs (links and assets) it contains +// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { - URL string - Assets []string - Links []string + URL *url.URL + Base *Page + Title string + MediaType string + Children []*Page } // docrawl begins crawling the site at "url" -func docrawl(url string) []Page { - var result []Page - return result +func docrawl(url string) []*Page { + // our result structure + var sitemap []*Page + + // what we've already parsed, what we still need to parse + parsed := make(map[string]*Page) + var queued []*Page + + // create first page and add to the queue + u, _ := resolveURL("", url) // TODO check error + queued = append(queued, &Page{URL: u}) + + // main loop + for len(queued) > 0 { + // pop a page off of queued + var dest *Page + dest, queued = queued[len(queued)-1], queued[:len(queued)-1] + + // skip loop if we already did this URL + if _, ok := parsed[dest.URL.String()]; ok { + continue // TODO handle + } + + // mark this page as complete + parsed[dest.URL.String()] = dest + + // fetch page + text, err := fetch(dest.URL.String()) + if err != nil { + continue // TODO handle + } + + // parse it + _, links, err := parse(text) + if err != nil { + continue // TODO handle + } + for _, l := range links { + u, _ := resolveURL(dest.URL.String(), l) // TODO check error + dest.Children = append(dest.Children, &Page{URL: u, Base: dest}) + } + + // add page to list + sitemap = append(sitemap, dest) + } + + return sitemap } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index eacf080..e3ce469 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net" "net/http" "os" @@ -43,7 +42,7 @@ func TestMain(m *testing.M) { func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal(err) } } @@ -52,7 +51,7 @@ func TestServerRunning(t *testing.T) { func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal("request for non existent file didn't fail properly") } } @@ -65,8 +64,8 @@ func TestSimpleFetch(t *testing.T) { } desired := "\n" if body != desired { - fmt.Printf(" Got: '%v'\n", body) - fmt.Printf("Wanted: '%v'\n", desired) + t.Logf(" Got: '%v'\n", body) + t.Logf("Wanted: '%v'\n", desired) t.Error("fetch succeeded, but content mismatched") } } @@ -74,22 +73,40 @@ func TestSimpleFetch(t *testing.T) { // TestSimpleMap figures out the site map for the site in baseURL func TestSimpleMap(t *testing.T) { pages := docrawl(baseURL) + t.Log("pages = ", pages) if len(pages) != 1 { + t.Logf("got %v, wanted %v\n", len(pages), 1) t.Fatal("got wrong number of pages") } - if pages[0].URL != baseURL { + if pages[0].URL.String() != baseURL { t.Error("page URL is invalid") } - if len(pages[0].Assets) != 1 { - t.Fatal("got wrong number of assets") - } - if pages[0].Assets[0] != baseURL+"/assets/image.png" { - t.Error("asset name is incorrect") - } - if len(pages[0].Links) != 1 { + if len(pages[0].Children) != 2 { + t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1) t.Fatal("got wrong number of links") } - if pages[0].Links[0] != baseURL+"/about.html" { + if pages[0].Children[0].URL.String() != baseURL+"about.html" { + t.Logf(" Got: %q\n", pages[0].Children[0].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"about.html") + t.Error("link name is incorrect") + } + if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" { + t.Logf(" Got: %q\n", pages[0].Children[1].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"assets/image.png") t.Error("link name is incorrect") } } + +// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +func TestHeaderFetching(t *testing.T) { + if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { + t.Logf("got %q, wanted %q", ft, "text/plain") + t.Error("problem fetching filetype") + } +} diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 4142836..72c5c24 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -1,12 +1,43 @@ package main import ( + "errors" "io/ioutil" + "mime" "net/http" ) +var ( + ErrContentTypeNotFound = errors.New("no Content-Type header found") +) + +func fetch_filetype(url string) (string, error) { + resp, err := http.Head(url) + if err != nil { + return "", err + } + + // pull the content type out of the http header + contentType, ok := resp.Header["Content-Type"] + if !ok { + // no Content-Type header found + return "", ErrContentTypeNotFound + } + + // parse mime type + mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return "", err + } + + return mediatype, nil +} + // fetch takes a URL, GETs it, and returns the body as a string func fetch(url string) (string, error) { + // figure out the file type + //filetype, err := fetch_filetype(url) + // GET the url resp, err := http.Get(url) if err != nil { diff --git a/src/docrawler/testsite/about.html b/src/docrawler/testsite/about.html new file mode 100644 index 0000000..e69de29 diff --git a/src/docrawler/testsite/image.png b/src/docrawler/testsite/image.png new file mode 100644 index 0000000000000000000000000000000000000000..db549f48d5adf6e823769c202a808700c31b5729 GIT binary patch literal 938 zcmaJ=O^ee&7!KQu6!stpew>EvWd)l_(oNfhW=op1Zorh%4cen=GTjc{Wa4CM)1DR& zg5clStAD_Q9`xW}@ajbb@ec^=q}z5qSaZm{@5}Q%@5j9Jrr&$KUw%*)1YzGjaR&S> z?DDNW{@?2lfAHfz>kQc`8MApnkzhq+gn%0dXJ~+eXz}6;Y6?Q>5c@+m^qv?YiKU>x zNO_!cwjeZHc^ZUM#J~uh;lvbwz5gHr9GT)X&6B;Z%>!OBUAJ~Kk#C@V*xu1ktmtLybgK;=G^ z8&H)ivTkTqLsr516L~fojg5iRUFYJSDNYzm4G8D+xiqgzgq}e~*L5hXP*p1&QOOnw z3-U^m9c&aFl!X+h43h*DiouA?m?`o~uUm-It!$F5$HWH)^B{$aBo{4hfS&h%XdG|R z85^LRdjBb$`HK|60m{gXhWz5j2SuvXuqg@{p*|t!n<@4ugpq7QQefMwsewZ;2ys#b zAFf!QXShkmf+R$)V~Siy!ZUALT?t|*-&t6OVZ)rO-g9i=O`wGG!H;Vec8 z+i;^BuCJ9eM!Rci?BqgT3@onquj}4~_zo53 zc6qt8U4NADou_=cKYyn``NO#9I+mY*9sRz#YL$|YpDymc_3?#J5-z`$Zoj_s^s;z8 MZl~wGYd?SW2WXWc6aWAK literal 0 HcmV?d00001 diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 5381b64..38379df 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/url" "testing" ) @@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) { } cleanURL(u) if u.Host != "somehost.com:8765" { - fmt.Printf(" Got: %q\n", u.Host) - fmt.Printf("Wanted: %q\n", "somehost.com:8765") + t.Logf(" Got: %q\n", u.Host) + t.Logf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { @@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) { t.Error("resolveURL failed (probably parsing)") } if u.String() != wanted { - fmt.Printf(" Got: %q\n", u) - fmt.Printf("Wanted: %q\n", wanted) + t.Logf(" Got: %q\n", u) + t.Logf("Wanted: %q\n", wanted) t.Error("url resolution with base URL failed") } } @@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) { doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") + doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html") } From dbf49d259b8bbd4468238490b03ef135cf9b2d05 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 12:18:00 -0800 Subject: [PATCH 19/46] flow: Created branch 'feature/add_test_coverage'. From 12cd1212714b587b798b7b88beb315cec8e0fe0d Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 12:40:56 -0800 Subject: [PATCH 20/46] add coverage to Makefile (make cover) * also tweaked GOPATH as needed * re-vendored with cover tool included * ignore $/cover.out --- .hgignore | 3 ++- .../darwin_amd64/golang.org/x/tools/cover.a | Bin 0 -> 97334 bytes Makefile | 18 +++++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 .vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a diff --git a/.hgignore b/.hgignore index 4e3c3f1..17b7297 100644 --- a/.hgignore +++ b/.hgignore @@ -1,2 +1,3 @@ syntax: glob -bin \ No newline at end of file +bin +cover.out \ No newline at end of file diff --git a/.vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a b/.vendor/pkg/darwin_amd64/golang.org/x/tools/cover.a new file mode 100644 index 0000000000000000000000000000000000000000..6d8873487a33b2202def1f69e13f88a2927cc573 GIT binary patch literal 97334 zcmeIb3w&JFbuWD8ST?dTl46YG7~)Knfyly1X%K5_fB})>CB_p`WZ!N|PrMrrwOBN6994U>KDXlDE6A=vQ?AQnSVTWv9>2CU zPWg!>S;KT4(=K*#9Jt3_BN5Z6P*VuIXw8~<%X@%Bui4u8 zd*a2>(bA}nY-o6>9B)~}#B1HYwRC%1mn46OKg6>iuVlMAI@>w9W?p&w63YS~weJ{+ zH8AzJ<(Ggm+wx0^?A!A7rLnerlORF7BnoZIFGJgi=*1n;c3a*$Z8u{a`tMQ0+16Ve zwyd~it;kK@l4#0J+>(eNv3e}as#^-c5056TWL*={bQ=iHs;5CjT7usgn5;#1llb@E zCP3V@wE)7;@1WsnA85}Hc6aAH`v;Th{9xBWe@AB~QylD17xS4^%F>w&;KN4H_{;}fd*t)U1^F!sabytBB*A0{kqxrjc6i3&! z-BlbOD2=WI{jA$s+L0gL+EyCfx^DNnGN^xS9SfUvBZ4{GwwCPEcEV@~x@2t3vX0x9 zb<8g2CYN4}Lap58`KG(i`3c>g{Aq5_i!Hgy-b6AtxgnA1#h={9?DW{!dk?IAHaB^n zmD_X1D(70+ziJ`}U+d-2C!V%=%K$0V_V_Krd}lj$#Z719IgRA)BZ zmF!G)WRu0tRC|B=jYeWO-kk><5sCeMFo~_qP2QBq5S_(|&RTNW!(;d44qWutxyeg& z6FWdMcmXQIi;0U8xyc=go?P~8V>!GzAwlXja-wu1!j@hDT?C+$BAaA;3A*RGl}98^ z`Cgi$6WY^EXl;s`?j9W~7yI+0qeI2fk@6@=>f=mOApoBVp}8kR(hi5D9Scd@lms2G zhBV0UDwoPTM{Z9hlPRmwjx#~GB04?xD=I(!SWC6tLH})p> z;YvTcKd9Y5qTNqv_fM$%$ww2raWBaD(Zo34-;z*7PaFc~VK$2ACLT^4i~)GhzaYc@ z#6H}Y*GDQI&{RBOprQ{7p1S`zs{fJP#Ag$SsPYqQ6S)KDQ9FL~j+G4Zfe++9GTr!* ze|{5E54@F9xTfFOkf^5zfkw=_k0y>%G8!asJ(@Vh&kG4K^|M^|CyjeQM0h6miLe0Qaggjyv@imNXn!I}&+bY$;T>wO8SDk=t`c z0;xSzDEdWI5Q9q(Tsx>LO%mOsG8d$v$aT6Ax%Sk7094Txve+wtztOYOvSOkKMW+8p zW`f9-EkY4o9=C|fqZDot9>gHi8&Ty)6H`=m5OEHjM?bpXr0xMXMeiW0Wvw6P7mZJ$ zknvxksgExfSze*M_)&J$P~7i%6%cn3#DP2F5Z)N}A4ruU_s34Jg4Suan0Pwz6=WpJ znC9!#2|`NGgple?^&$mr23at|3Qiz}2prIhKhSwLINc~K00O6isaN9ME61sT zZyd&7Yy4HK+zSR?)cBGANN*~tk!*yyrH}&lkapaUwxyg#Kbm;P4(H%q^H0Fx)K|iD=#i>b3ZPlgoOqwIfT)oM`n<{HxQz490nLJU*n52 z;1A0I|E%ADixOuy;A(95909k0J|=mJ$sM?W3G?4JwfqTD%LJxSIY@h6jS>CR05v|i zr;hp`t)4tLH@Tg8VH=O=XA>iI)p!eibd&~%#WiR^NFERgP0rJtX@?_cXk|5dA+hXN zkcZ~hlZ_u)Y+{tG!Zasmsh?@ofg+A`dCn|#gouThjSItw(*swNW(|q%QJD*)C?=fy zDHqRDfEK(Vvl4?)b5z#7BCAt^6edm~K+QS!a-*n2U!V#G^^qowFhc?vw??1Rm}&Mr z&0}FVKFE~3=SVzvKo>l+T_kl-*FI{qo^L_pgTG6>wdV*Hxxx8*-1+JnAG|^2>j}%Z z*`04;-5b$gTMf^2F}DUR7)Y#0c;fPA-#-eK?Wi2b!H|&gTabv(WtF zXs*)y$~2GMBuyY2C;oclnP)dnoXSo7GB@$P-1c82bFp9M4wMtyxWS}7P#L{o9f4kA z2FVZ`=BOUT6y+{rh&aJh6G%&ds<5Bi(Mxil>LePYREHCP_=_`VChFKqbMjeCLZ>QI z2^ZdYZrDZUJTZppNZqIC{+ts_D>KxDePR!ap-HozI)Unp`zCddZhM5@v6{9M8h38; zy?wdKQH=VN&XjqSsAZCMm0b2j<44*Ej#S+V>Uo*K3;W8w(lQ19idytC#AE8uQ{SdJ zk7`5%N425rI9=sBXE*NWWSs-W>?pMmGi4ld%b6*)xtdk6M>&}YSqU%kMo8Md-y|Fu z^%jzSuw&qs& z$~|}zG)JJQqU_xRw_s8@;1yATJ3w2h zAawN;a)7vEsDNmStQej}Yh=antQsV)$WM6IoMc(FRqrk|l5t^dz!9p&G{fLQG?#kM zC5vt$-A~ox7RV&03b(+3=9CYmmFs^rcLru!tlPQUpxW|VbyZL*UE)1(5fl{+C{>DD zPc?LiLSTT9W{j%Zk0uVvXHoM<6OYJehC{v7z%m@3Et$@6c&?}CDpYtqX;G+^8jBH$ zgK+`JZu!i(@I1~qD{Na~4G)_zs_JZ_6~2%8v_c4E&!Nhx!pWvXf^!O2`q4cZRp@t6 zbf+||K#+eNMk8G6NE$CV@xOkBc0#_E&l9rshHWR#yowb2d?L=C~0#%=R3QIhNC8H52_W*mBx{U7hPkXn)3f zasg$zM43_tii6mjDwcPRQsPe+)Hfgz&@HXBud}A@svV^`w`-Hx}~4!(5%u)*IP|i+76+Yh?RAlAc>&sm9#!6jU1r99fj#m*NEz^9i`_7EfYbLR%)SB-jVDS@;K z{4K4rMQhsV21UGZo`HdL(WYs+vVXgYvO7&L-s$A} z+cyR0dSF>lu8%IKTq}6Y98^y&Cvb<3Xk+rG^wlVR>H?$ke%<@|3kcz5TIKum1956V zdOQhlnKuy!aug_O!; z7kO+fb^Bm`tcY;0)ja0>1E`a3DD2~8@OHe#RNvYu*jzxm7>_nS|&y5Br_ zsr$_@Ja0a>mMTkb1n}f34;92dStrTm&#$dt^unr&C?P3YD2?2Mr0F)wwN?^#rSI8u z1nFN|ML za7fz6+Jo!zr4H)6rI=c;zWy3?){k7zui?2#OF!62xEyi8(^Tz;5t2jrBNrHi;iAJNg;a3pI+@( zsO*TZUQ;QQ`rmONLPz`!rv;4Ni=3F#$Ii*?9k@^a2GzkK+{(Z`&k4rB{WxdT*VKup z-&?7#)SDdT^Lw0j5tE{>t^S?<-UuBSUGeGnQL$qBRvIy0|95`v zE%Dp@K3s?XabRH&_J(x8$Z>2GJQg#Z8M@Q(01%29snT5?mMa}|NoaZ!=@oA7dpb>kPAtI@+z57KH+&t!I|VlBASSt3?+kYMIoRE2}(O zRl4AaK}3yO#bVV=Zzmpc%K;Ziw{>QLWEE4=y5PF3Zz(u)YI4C*C-pE1U>?C@y*TQ( z(syMd_6XItUGVMJ0mRhoLghE}CXXhq}kG)7}- zV9U~-)-xfg)Y(Cuk|y2^ObqMJbjtd|lB&WqU7=XHWHEa>m9-wPjwMAs6~c)rlIy|J zIMro6W81D(ud5!N)(>J*Fm6eQNvTx3^{qu=g{7@8gh~Z?BtNuB6_pArDw(yu5InBQ zjP-a(YM1qpZJU-Yv&l)~Trp>wDvS%P!+K~*6u5Tl%wpNej&7oyyuY>B=&|KO$iak# z&N|m$@~Mc4HpF!d4(*0Fz4cPPNdzkeNsYA$M2pQ-RrX7+_p+b|k>e%597~X6aM*gt)q2XY(|Ts1 zo3I=3p6n&o7hD-Afvj5Aw_FK?Y0P>&U?6DOKM2$7nDwnC;o?m-?OSHksQx2^)|n;# z)Il!YI`4{cOIdLl24K!U)caH$im*T&fFTs_urEG--gN8-=f#enx4g-2TGSY8{MllD z{9GMhK7TG>PSx|}TL87;A$#x=_zK^gw>#|ydwty{b(h%JF8L*x$9|_>w3pTWGG>3* zu6yTs3meW^*s%0Udx`z21$8a<_4fITJ{hy??X8P4w)(H#{*it8!Yi6$=T!b%{Iyv*Jv)9C~e&@m1+Gk?`x^uzqMaAqxu_fsRjVQthpBNm>WV_OEW6!o{5JEE8 zk?n*(a5|mp=tw7vUAhNwe^-W_-&4g*wj&LH;e1y|x-d{EBs&Y8-Q5FZqnUdT;1-Oy zxgf7Ss053rx`3Hq#g){5FV^&LEU<(O24>F)$A8-W+P9$MAnl4RN%q2F+ zo$QT$P4`1_z%NwbybDgAr2^lc=aQk}0Lz?Bt*UV+k6}|Qz>N1=aMD-dt{;%=+aaF( z$3xP79+LK2NZO`8Pjo=G)4DxiwMH^KO5{rjG25L^CkOG%3?hnZpuM}jt232>Vtuwlth}5cqpZLQ-_tk&F1)Ao zIDvmaoWMaM_etqsGl{)peuf_iT_?VSn3u*qkI}tii5>7oWA9kGg^Qhh2Xt&i1B0Ug zInfHR1LVTyIDyEl{8$D$x=%yZt_ceVTj?O3~mb?Ess_zl` zY@HO0%6=<2>Qu~H{{TP%N)4oJ#;lb;iLHWe zxyVk}y_v!d-(?pJy2vLp#VkU@3jLYE>_8@sU*|v(fm+3mBm{1!E^ZM6j~K*!N1>yC zu$a$hI+DeXOfuV*Di*r>ll@rU&x1;q1W&cJO0HVdRtc>}5XD>QUq$yoaMp41AaVs~ zO_3WIoO8%$k$!ZK$;Y{W!MWd%$T0xCe4e08?63FTg}9r%D&9MJ72HI(vwz+rNKJj4 zTszUlklcv;;_I`$*~9l0*lB<7Cn(>6&&nVt2vvrkBXQV5p$B8xHNZw3Z8(AI&R>>yS<4#&fCi!i@eqPz@66Bsc&DM`tuy7 zu&+>R_Hg5Y|AzUGUCQu+d>S{xFZ95t^_Se8yU8(f@_li1snMOeiN7ERQ0hl0hvXo; zjrx`Pmo>#KoiF_gfAvI?|A9sU9piMohEsl(e|2xezcn9iC2 z!@cM;Aag$!A8?Ju2scTNg=n7~i}7k>L6Z+iTgB9C!*G}ee`bTRS9ZGU?1!CB7E~j| z8@Xr;!cl|qurp{h7);1;Z`T=j|8Kx4kp_jS6~w%m8c3tY4SiHZ&(WaC=WTRVo`#r& z)h(uB?Vg>J8C7+!%ln_30?s2`s3evBCn?Mpr5-{`uaq;_Vm0`#_a3}IkP2zNc!BczHl8_r;x>3$vdDEbR=8f?kCzkHi30gpE;y0ErNvQWt2#4& zFIXYn52Z6ejr}P59X9d#7QLmdsOvsxEFxp^NTHeV9}hQptMOqEex7kE zZDbn#)nlv%QrO^7_J6;P~f^A%b|?q=E#~A#16wtFxdGFpbHK= zH8J}(hJ#)z&g{7+PL0Bu1((OKOvYPrkDFAy)j2rF+yzWw2jT}oiyyN%Y#=-Y<6sbh zxPW243)%`wC25DHqT{-9(q3_3B8^>OA`M*iZcEZGmDsn}xyx0HG?t7JaHVLwyehI} zaIKTsFjs_Zx3w;CSC7^P{nj%J!k4Zu)oC3@%h@q3XBTSkXxUM~@h4|=K*#R?xDHD8 zaw!<@vChQo_cAe%h-(3%t>rVD*3xNDYv`l}M{vDWT`rKz0 z#ikeCh1J#cQfoZ+F?(a(Vtco}%sy|y(xvf>au?LEYO>eb1NirISn};3+jjkVZ)(2J zu7As$mb~3wU@xva&tCA>h0D)*Q^UFj%Wkvw+I8Qt>mHB6<{pEQ|Bq~Y_riA=i!|85 zItP;7sbW|6;9y5*XQ8Xmj>BP!1KC0$lkLyyi?nPWt2i7?G?>cwXR-jwWU}q)?#|9) zu{+a=&0P@NoF6;1*648|(Pw5k4-_Dd?fjGt&Cu9Yc%vW6)aKJW73=u}n$6I#oHJ-G0SkHg95y#`aU#b2u zrcWHju?AyTz&dEN0@WP76ndVpf z%@Bk2--e|9BqZ$@A!(b?gNtghZ3ogOo3w>p`(?6j!vOckG5b=3O5gByZ9gg%mhCAl zy&tKo*jY>}EhFkrt#}lb{w2lFX20IJhhiiZxlP_`fdUap_lS$=8Sj8t}bfj3)I{AvJ?^h6J z{VFkb(Y4#P3f#-6VDBb`Zd4}X8jg;dDOLY4#EN|?B<*xa+J0V%_*>2UFdnlz_^cmd zIfO+J7zm4`ZLRz#yA^xbzghRbh07$TY&SUN4R5M;E!>Db4hFHuo)MpH{N#iE_IgEJQp~tQRnzRwC#}?trc4n zgIyhMIG4=$A^FMhWE}11ac)8! zS8tq~L^C>xltBtw;t0dGYj=&6uPW^vfyWin9%e2#A`=g>G8Yo}@ig-IbfT36z7_VW z^2YvE{Tc)@|Qoa8}jRThF!|f^g<-+m;{OMw*+mnzpC0 zdZ*cxS)AZEd4s-B^|Rh;ufIqE3~ z_$Nbuf^)2;9DLG&e-62?(gV(mA5q7j{DrzXsAA!N3ujc+up_&diOVq{?kw)4zCw+E z5|kiRH75sVCz+$C2NQ^C0b=)eVL6Z%HJ9bAolIONKxHa1-_sgo8>SHN+cC#Hp$bvv z2@wfFz9`Pf!^fuQvDkZJJ?{j8AIAx^b@+o><4XJSm_gg=fdV|}aIEaWKzp&HzkQ&X zZtud8#D)HW?#}K^yFP1F<%9V!Cy1btw&I;G%Enf}SFvHo>Ikidq3H0^ffoUB>qz5M zAEOCwaq=0FkilAH_u+_X{HyCTtr^)uce1_ES?KQ17E-X>6tK?O4&URkz zAf?wiCs5Ac*rtwG-m=Jo{|3u2Y&MrK;A44k%Ht&MWRvG1IRQ>AIQhR_T58z*{$ZNQ zo3&YwkKLx}>j|C_*J??XUZM2A;q=v7I&5N;Am%+sHD>3JyGQvfia3#j5EMxeRYgxE zt+K7WZ3ufUbuxShZz=^^=7C1w<3opF{vigWck+B3IVS0Z4xfgM*-UhV4eGxh`_RS%YoB)xLmzS-hK?mw zlGatA>F9~?3-&yh!#3tLq#Jzfa1#2TOG3u&@2T4bCpKgH ztHDQgwq%R1T{s79kQt~7-Ts1CT@Fd>N3nlB#Om<(A!$7a(IZQQ3|%L8TTGG{e3JG` z={t-C8KD@ejhR;2s}9CqjXe~L9gJNHewv2yViEpeI=#<^D~xV`u4Ot+cBV71%MW&U z;Y9a*cd~nMAU&8&raKGiRCgwiy(i0##8A?H42JsH!T`(Mo)E?o?g`*BFxKZT?vG55Gt=8m-W9-kN|;?6s)>$NS%FqRk4 zy18X$+m ztBhZ?NvZm)U{6>mRbY_RG%era^~`pp5M_UIUy;-uxTYaQ;%M&K=>@qzd#3QvpKX8X z1}oS2g%`*T$x)IvV8^G4e8Y(KXjO;pHD1-h6R@(XlPRIY4}FRkTx=JYgb{nq!pr3rg!XXWh6%fhW?9D7f%X^ZCr8xqrc><|TPP7~Y6MKnCxpB4vX zJL_l>UNYyLEzk;e>N{Kmp(w$O*^9%=kMXPh@>>TS-a#H=pQ6`_#l68r#N{Ll#_n=} zT+yWmaLO~`5bRtDCuQA^mfA3)-@96G%y}X zO`rUkv}DN(bg@gIi>+`(K-C!1PqH=D*AV+0Cxyy@Qw*ZZ0AbJ?0(c@R|^&p&EpFW z+h^?9_<{?eR-$S34fum^#!bfTe>Nq=T(6~kBt!>$DT|HNEv_?iYFdCp+(K6TTUvm( zs1s&~hj5Auj?)TWg?Dy^tio|N+kCCU#Yq~i?t%HWS&JtH_0d|~LwnIy7BAu(s=fV0d$h(qQkXJi15Qofww5vm9LKf$p-LqYs2hh1%q6Z}Z8ZOigaqK!B zttN47|2m6f9O@Zt6zxjSyEt|Y%&((3b~Q1ofmD&*zl&p!eYC4Pia72fO{1zf?xN$0 zf^4NY5?9|(nt`oU$y*l@xW%z6`PWb!r-j|rjMn`>-E!KcTTaccX~wKGZ?;pREVAv? zEY)j0Z8~e#(++=%sn|~+)@&|DDAAr5&xV&+i`?6_*~vF{jm)D$2jO4@l*cG^FkvQ@o0PGVLbHhBV5a;>(@a^x z8!3XY4j8_uT=||bQJGKR-(TUZsyXyIs+oV40W`!L^?M-<^`nq9@wGI6JLjt*cFui^ zf`!5-LehG`+)K>;@@_g!L)#ml6JNn_c`!-x;De9{Q!r^R#~RjYD{VY`D${wX(b(0c(_PstKQyb*u-y~EUc zin@OpH;U=iJG#<)zhBUMf(4#&>$L@b*{Zx3^^a4|ar&cv_2kDX`gB~qqicOL%8JvS zJx+hsuf7>~aef--zw(dvUVqIp%Xpz4;_9Yyed3M_6zb z)#FN>30O^jbJ*@#PX1NTsdegJr^wBqwyJ01I{9gGB)Y(aUc=#bYVOtaw5q2z**%uk zL&e4M&-r2QvgJSPKjW>@jH;(b**y{`u?!94612sAHF5qsITs5!jK00l-`&xf=^iW= z2l^2?JBVZFg7xTGlJi{XvQ-a?aCBLs1&DqAhq0f-cz=5;hL2j~pZHxNud_Ks0L(|~ z=p0tk#Y3~9_geF|>W3XP$rw-fCyh_zL=8S(#Q0oopT8dv$tCxZ67o!OiYG^eI!3sK zoP{bMvP6g~eeIj+S&k6*GA2_GAm?AJoJjaM3-KxU01VF8S~XC{!6t3J4rBbuai2;g z+?AlXQnaE~)uQrM+<;2>TP4sB3WF277=XZW6>|Xn+CbLP00gc!2+KIy;C%2||N1h{ z>I^^=67F@n;S%)Iru^twQJ$P0)Sq+`qXEZlS&s8Ut{(}c1UgQHk{Q++f9oqv}V@=ytJ4$0XL&cW_<8C<4?Tj3w9T_d*NL}CT&d51{oZq1c`tU8j zEEtg6aXOJ;qUMTf;K#$CP7TQH#E>XBQL^|r*>%Oc#V1_@GOu=lnXPs{*Bg-AX|)4r z=6AjjUM;@^$oVs>f=vBdqF;ZhLK4*M&KxT(iBpByiMdu4$Jvv$%9Sq2}yaK^FdgbKT(s~1* z5VWT48l2pGV;LXf-g@J%ejH&`*cO25nh06BjLTGu!1wt=0fFwR+XwSwWsFh)YDY?Y zbC?))z6c$32ctI!V{}HHN*o&-+B%#HsGBqD6-dTFho6;88;8pQ`RH&7PG@LGo;oLp z2V`?3iv!5{om$ARwFl&OsHE~y_tOo|Y=Rc3Z>|UAcSuO;ama6kpKfs4 zAV_#)Kz^qU5{%1V>3ozuAeV7?`RN9VO*5j6=tPx(tgNl+QEcM$6=)9>>0n|BQ9(TM zS2DiI7Y}U|1ysk2N9;NR&uY*-^-3Fk3@0GF`-3^Ie#ggh0x}x?F21Rw<(1#lsg}3$ z*`3o3UFbdE&&&iM5pgBHpQSU+lR3=ey1B~<4ufpTXQQP-<*W=5-sKG;Q?9EBuGf1JV)NhREg0y zA0gEDjFClD5y;vUKC*7pPb5loe!1M$yKdLc8_PS(Xt7`QWU5c~oP2~(4=lV2vztVR z4J7eH;5Yv`O(OHjL#UEk*XjH&hU7Ad$Rf`L`spANaWDqLAHW)?P{mUV)$si0|G4Tg z2$TOL%9#0IH7e>lb4+GVMUduO?^BcamG2w)qD1<|BSBeKOGG#YwiZGcyE95Qxn*w6 zLQ6{A>QaRTuJl*;Ofc2t>~zcupeAdY#c-h1InK!y4}8dNn|dHfq8#E>9sW4K36K}# z3kaDGeu&@r%Y$Pc=jmzCZ@LB>Ydjh zt~OcLdsVjO^2QHTimmDxNqmXGVE)9FDf)yqq!@Xy&w_7l{hL+!Grh7lR2R&dUTvsq zk4oN*(p=_FQkyh&$lG){fy`VV8US|EFX6znz0GfQ)_Du`*;xr178)|YKEg3*pgj7| zkNPCl<#K!X>Xe6Nt44JO&57Aj?Fw4_*#nHZ0VC-w1P9NLS`oCTa9w_XFwno)_<4Z7p|wWR^Vhncoj5v6h@pd^XQn}hJ^@{1P%I9Suy-%cb-Tz3*av z=5rUgn9}{JRm|wYQUm>0QUL1p=?HAKY9hj1+Y2%DZbcR3 z4naQ?NU~8F7B_otxs|uv1K3lh`mm>yj}VFz;vHaf!u-L9+cIzL2~!_KtzANc#fTVl zr1DUxIx{Y3SoO`t5zotJ#ST4f5(rE7DFqD%;v1@ zst58JWv>!tYq9rak0#@+^K}8AjuX7>{R9OyBgun;GbW=T66&t|ynjL?)-`QdO_`O6 zTc#r%)xXXn#XGDGSWvKT9#}UjQjepNT(3Go$c7#&jDC&cqGW@Dl1vWF8X}r%7D;l2qm)RQim3U1@Zuz~oU!DM=#o z<{&(}GGKO~M^#qI5}$uguA6tjjzNh$Ru`B2^1uGWiQffT&)PJllx}sE=x2R7Tc#;h z@dT7jXdFUHxkVpL3zRxjdy=lQcqyLw)lV)}Tc1MV^yAZMUe)^ZgU~m1BaAR0`BJ>l zHnng*#Kf@qNQahYTgViTIY{V`emo#dM?wcsUxTwOD5Fpv{hA~UOF`6s(6J+VOgm(B z0w4I{Jl&AdM1xwwD~1Da!w$1vQHNUMxWOb}hGQcB^!pkz`{qu?ly|Gov zYRzOyrrAlGdCae>&v@#&30R3Y4SS0kEMaR=PnM6?GltQ)Ysc@TCAA`X#DAtt2U^jo_wP)s%!4w0j zt7^4+R#)|2U6;)pW7XD$%t#cwrOgxWk;1D+-2<`~t10Q}m z20A$A(1G=dPyGStz~KosE^~ENG-P~J!7u3)7Zfa$1O-H)AR#kr7U@Fi4)b)JW?>u# zVV(&f`@0fuBy`&iiaOMp03J9g9`!Q;JTNqTrXL5ZNn&a2S!8pXM5`|%WFA7LcfC>N z9-qmi0w9qFe_&Tm5u@kL;c7$GPop{Ya9mN0KA4fURj(Dfyau#`L$ti@5IJ1!sQLvc1T2c9 zN0idb`iRk%s$ZOPa8O^6<(U3qTT zoWEl5_23274b`smXT_g64ni! zw#0nunmu)Ej!7PvDti#Iql}V0zpa3kWfR0pQ0g(UYX)vp)*fMnLj;&G$p%h{;6e6| z*i?ZqJ49PG@M;{rcaU=sn7gKY`qET*ve)-Q+0BISMF3A9XROyms3vN7DxeB%xI>fK zEyLRYnQZ!Bxb0tG~AHda6;PTNViO!wxkR7t0%8qIBn#^f`kv ziBzu(AXRTE(-4~#v)upq&wn>HmsG3fZFN;$p{P||)%&39{F&uu)WJ;9ebors)7epI zJMzd0GKWpBh8(p~yqK7E9uRY|YfVAsnNR_(^@HGuel^T-c~gxeBs0`j67j)?>U`i{ z^FDs>53hbbSU~kbwd?$?d28)Xruv&}pn5c|`U)n>5IM9S4S2R>uADxN=}9epl+)+; z?z`|=(8;U=ZiB{OiA;43f;(I9)3o%Nmq6iaIw8|x!15!6-o%2(&visBcl;HBDUVx8 zW${4@>LE^V23L4onB_vrAMZ@pN!64ZA)L)_v$^PfIGKwZ%g4Xm)wR^ybL1&B zBM%#`?2eAkt_;4a)0Itjb`E5+?U{j0GMmriKAGug@3$NeA7za*>pr8H#w1f32Pu+L z*0v=R(swctkv@a20a6#rSVtMp7>Y!QM~Y^wfDsBI;5nADl3>T-4CJ2iL?mNy&1D>8 zP@SkSuYaTnNU!`g4hg}7b7Dh|eomDTgQ=;*PQ`@)@Els@q)Q19ke6zBkq`lu;h2UD zn2wFhlHY&hezM&wk3gqBGrdtH)qq(Z2g3-epx~-Xm93!T(JGxL?mS&*iQ~}%Tgzgh z%_SC`me(#9#2oI!>9p%p5f`e2b(J>y2n_c?(BSAA%a&WE6Itbi2|s(78%$C_I3_Af z@C&j-J9Un!FeilXe(m?jTY1*1>HJLy)tbo^;E6}ln;d}({8T-X?)sR!1`AV#UWm`| z`*pv+1j)#soBY+YkRbc;GnmwL0R}1hk(0*-*aIi&N#92Bz>xl`1T7`%6eYw+MArWb+apQFBKy-4PLJbXTQQBGBOmo}hgOUF+KbDdnE6aZHuY zVnRycSSs;T;_7q*wLy?pTeDBWVwa{e2ZGxAr7&?cxN6xXvDH=*HB=JQsAj-L{V5|>{t!lti+N_Q1U5kGx*z{*f1ON{Y9utG0?QE`MSbRwSYJd#0M>Y$*z&NwHrZN!k+u z`>eb$IvpYBRSSu801?G{QSVI}qI{gSI zF&#?@l2nDO6X#^AEXFqWC?+5I{{rFht~3AuWOlNJxx`$WAwqH-MsJbVC@Q-lRraky;?Ur1FTk ziRil&(K{8;#=!o2m{|VaIdA^)!ehULGTN!CGuPK9d>H(gHGlpZk-LRSHRmB!zb>ju zrU1DtNyjfPNZY!VJ6oppMOP6JJz1z0G}5K;AL zw55HX=iB6SQm`MWr0D7xsa%S~Ry<{V+D0gup0i>N7+F#X4k|)3PtWG4zZnJ5 z&QaNgbhbO4>h5eGNF}>E+PiQhZ)bL}Ki!#b&t^M22P4rO^(2?d5q(WH7Fda$x4$UJgQbQRu zKMT7EBX+Zy@6znP>EZk5Qh!zYsk$nGsh-tUy;s-yTbk9>fh_491HjjGXUkvCEEOWK`T(L}~KRn$y38N2Ih+0({~ z?80A+KR=fgtyBVaReho;SzXoppzHjt$Y$EjOzeZ^wC+pno-U6<_q7b6qkh(wsb+47 zxELccS`nA`CJ5*;>yUyQ2YzCdm z3~@_s41oq#Uxq~-D~Amt!_NT+s87eZ<*YmC=}TMxbxyuA7(@|DzN%{w+}V0Re?dg?qvaDH;oJSZn%i1{F($ub?C_Uo<2o?IEp(^svl3Nz z1*_>=R9&iSKXz2Iv~6*LQ+J9uVD-g7H-yi?$oM#lv-3fh7R7Odmn+$hTNeH|e1rz|q%NvxuhM>&8x$I&Nq2W&%v5j)H8GI=$t}7fRm>q?=dObf`T7sszY7)! z>NZWVq-zr_E^r^g;)3uQ8BMUbfC-&W?XKin1dA)HzhLnI`MH-D@H3T5^GT|jZe7zQ zqs*d$WRwXd6&Qm@O$ecLN>VqIEU9+Jr664UflvQ`b19>+#N(e=M!D({BBQEwtR>XX zD(&}`QJy-CixsEeuD4k1`s$!H$tbhTbz975%ZC&E2adQvZ*!|BptHxa@W1$}`OO98 zLpv@AeZA63i!WDjaJ?+xi~sQb2S7M);F!A5&a%Wehg?e(qrnkou;9>OrG;=zo*oGy zw)H9jZGAo;Eep>i$M7ai*9p%chE(iMdmw{H_z#!8@ON+Q5w5KZSR&A>MAKTrx+B8) z3+oYlzKwt%txJh%TeX1_pNktQj|x%mJ;{MTn2vNg$N_l?t&GH~WSD|}SUJt*&)TPN z#pu0(*QSAO5A4ql#!QSXZ#YCAO!ZZ2I58)Za{IFURQ;28=O(%W6j)oM8-viH7tRX=# za{v;g_#k0;Lb6IjLLbdSJ(N$XrPjb@iNFQx&b}=m`!=8KfTiqlt@M5&sZjyXN7g7w zBJt)RJam?u8s$=XTaiRQ5Upjd!=HunhSMY_pFD(0Ox>PA)aDuc6wsYoxJlytod>oT zApBLNn<}#AtBNXB*P0OoWKc?EK(h0?$+-)^sWq509Ggb=RRA%b{A;N>IJBzpkgFVB z#qkh;OIApo)}`Wja%p5!RoG$JRsh6iW4J-s2*crKM8TF z7Bo`}Y4g-qJWO+v@0>-ZeIRa}+)M z?~%I$?ZuA%_JLx$y(=}CE%Xm`cXnsmD^bJ5dn;(j%p;Q65zwDxly~@u&k;%0vdQ7! ztPydwEJA>YIC}vL#ll5ZzHZ__{^37A1!?kY9Kl;_H?nlP*_>0tyEIf((Rz5?mIwY+ z!*dM42crUyvl1qDYS=Ny6g8k$q1x#oil-ZfqfwRHGZ=zv9Lv4k#02!aR9wvOC7Ts+ zqmX1h;|dV>{Wee3LEY#raoUDH?0gXM<3B1U=aSf!N29vRK5^E?YJIHU2VLjSgM1d9 z&3v#U8XtH%J_<|JIF^nNZO!>+!YEVBTsrHB<&Z17cc5l&0wZBgAYVlO`F`0skdGt_ z&7@S~o=s7?$L(BxusscO{+f9O%?wd;<;)R%&>Rk_CCl6} z2b?y=s^6=2U>bsXyYhL#tv|c`|DB8VOAw7HVXUq}aA)iN{6+jmTc0qkt~Y7bUwsIn zO|c)(yEqgb>(VsQwZi{AgedrMLqz_Dft+H4m2R4X7dNT8(q>f=XF|8dmK%%hD0(mq zQw*C%^%4DRid-}PHx%R;W6^C&wSI=M=K1U(o6b6dDy~HVhrrvuOmJG;)_$Q`})8FvTW;Z@}^2IS)H27p^B}quz{BqeHVRS{N%fSG?&x4(sR{SHC#Qb zt9q}l^A{UjV<)mqBd+_?cZ6{fGd5OykzmDC9U^pvSZrc?0LP-6PdAVT zU!82-3cOtI-K9oK?YIS#up5J<=9v~xRo@&&*xvG{jkVr_qgOvVmo%=_33XLHqi9=Q z)%&39{H^t7-qlS0bo^6w^7nLm6i%pZARRTf)@(E251}vU93!*>*D{Y(`LDZq7uet`oovrut7fT#TOJynz zTTz8jqgc|#S$sh4Suf<%J?KfXmJ22UdH!#gCL~?!DOyt;;>BDufP&(QLrR$GN-&aM z6?q}J)~gQASzK|glU?*hpw9w*PNIuzkV4THfPR&AR)4o?6vmn;FH8GNr5!wdu8E5! znIc^=32#%f;2rWNLQXja*IXV=z*^9?yLviXxemA4eD%hEeLbdMrKYN@>YSkK{Hd!m z=v)?We#dx(b##;|IO3Q&6L4gBG5zZLrC6gG`el$3>L-hReP8iqgG|+rYwXSKGY8pR z8$!L^RVc?l6u%}e-LNl@Uzv=zqHA%Jinrp+Hu-W_N1Vd&X)DmPHjIwJXKffgGtDKY zTk18aQPRgP^%_}@?t2DU%6^e1Nv=$a{Aw!(xy0{Ed=Qddcihk(45=UtDL)K=CJeXn zR2#kF(Ac5&99p1st*)E?WD7*bto=%rE~T!r^|QVxhox$Wnj`>y^T)CRo!@m%mra@wXGXh%C7RJ#r06`UfYdU7tOa;op(?iwAS;1P#RHY&*l&RcH!TLIv zm>iWo&ZcYwlu31*&9|!1np|myu;_)S@0fl5MeS5ox+=Bo5M1!K&+wF<> zz_O+IzQ-Fm>TVYO!=yiIl7046FrD+yt3^?HswHZWa4nRE z?+TzGnd(D9PCi0NETDCbOmo#sQKgrH3|u;CApv40=tFNZ)xUBgpMT{jt(MAFaOu0^ zgF9tpR*UR#OX+e*k(KhmQ-u;{Zvom>8HGpmlrIjQxW-DOvL+s)rX(Mt7C96t=pzSs zS7m~Ux`KsTBu1mHd9=8-xEuS~v^Z!}JwqP=qMt~VHS)_vb)u+Ptxh73t5yo3=%OG@ z^2m&B`LS(e;!q}v!(Se>e1aaufq;Ajhz^ZvW6YFP+*#a7=mJozc7d6B2fI3=xA_$8gEMsX=jT^}i4;Ahn&5s12 za6X~n0CIkLj^+(^pKgu-8M5d0H4-26P~eU1a+Akuwk?s>Hr3c p@W&TIuhra~0x*WqYyd`wY8^pX`4}eaC?H{{y-{v*7>$ literal 0 HcmV?d00001 diff --git a/Makefile b/Makefile index aefe125..6ab64a1 100644 --- a/Makefile +++ b/Makefile @@ -3,18 +3,19 @@ # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/ # build targets (which aren't files) -.PHONY: build fmt lint vet test autotest run clean env vendor +.PHONY: build fmt lint vet test autotest cover run clean env vendor # configuration APPNAME := docrawler VENDOR := .vendor PACKAGES := \ golang.org/x/tools/cmd/vet \ + golang.org/x/tools/cmd/cover \ github.com/golang/lint/golint # override GOPATH to be our vendor directory -export GOPATH := ${PWD}/${VENDOR} -export PATH := ${PWD}/${VENDOR}/bin:${PATH} +export GOPATH := ${PWD}/${VENDOR}:${PWD} +export PATH := ${PWD}/${VENDOR}/bin:${PWD}/bin:${PATH} default: build @@ -35,24 +36,27 @@ test: autotest: fswatch -or ./src | \ - xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh' + xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh' + +cover: + go test -coverprofile cover.out ./src/... && go tool cover -html cover.out run: build ./bin/${APPNAME} clean: - rm -f ./bin/${APPNAME} + rm -f ./bin/${APPNAME} ./cover.out env: go env # vendor # * wipes our vendor directory completely -# * fetches all packages in ${PACKAGES} +# * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only) # * removes source control dirs so packages are vendorable (commitable) vendor: rm -dRf ./${VENDOR}/src && \ - go get -u ${PACKAGES} && \ + GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES} && \ rm -rf `find ./${VENDOR}/src -type d -name .git` && \ rm -rf `find ./${VENDOR}/src -type d -name .hg` && \ rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \ From 67817048f93f8a15a4ba25284d078c1f301f7970 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 12:41:02 -0800 Subject: [PATCH 21/46] flow: Closed 'add_test_coverage'. From 614f05f1d701c70446aa92e98aa8fbef3695c363 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 14:02:08 -0800 Subject: [PATCH 22/46] minor Makefile cleanup --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87413cf..8517d4f 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,15 @@ # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/ # build targets (which aren't files) -.PHONY: build fmt lint vet test autotest cover run clean env vendor +.PHONY: build fmt lint vet test autotest cover doc run clean env vendor # configuration APPNAME := docrawler VENDOR := .vendor PACKAGES := \ - golang.org/x/tools/cmd/vet \ golang.org/x/tools/cmd/cover \ + golang.org/x/tools/cmd/godoc \ + golang.org/x/tools/cmd/vet \ github.com/golang/lint/golint # override GOPATH to be our vendor directory @@ -26,7 +27,7 @@ fmt: go fmt ./src/... lint: - # TODO golint is not working from inside the make environment for some reason + # TODO golint is not accepting the "./src/..." for some reason (from within make) golint ./src/${APPNAME}/*.go vet: @@ -42,6 +43,9 @@ autotest: cover: go test -coverprofile cover.out ./src/... && go tool cover -html cover.out +doc: + godoc -http=:6060 -index + run: build ./bin/${APPNAME} @@ -56,7 +60,7 @@ env: # * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only) # * removes source control dirs so packages are vendorable (commitable) vendor: - rm -dRf ./${VENDOR}/src && \ + rm -dRf ./${VENDOR} && \ GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES} && \ rm -rf `find ./${VENDOR}/src -type d -name .git` && \ rm -rf `find ./${VENDOR}/src -type d -name .hg` && \ From 0c48b94ece50604c49dc7fa240a33b269f58d5bd Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 14:54:52 -0800 Subject: [PATCH 23/46] refactoring and cleanup. needed before proceeding. * fetch() renamed to fetchPage. now takes *Page instead of url string * fetch_filetype() renamed to fetchFiletype. now takes *Page instead of url string * parse() renamed to parseLinks() * tests updated appropriately * minor cleanup (lint) * NewPage added for conveniently creating new *Page structs from url strings --- src/docrawler/docrawler.go | 22 +++++++++++++++++++-- src/docrawler/docrawler_test.go | 30 ++++++++++++++++++++++------ src/docrawler/fetch.go | 35 ++++++++++++++++++++------------- src/docrawler/parse.go | 4 ++-- src/docrawler/parse_test.go | 10 +++++++--- src/docrawler/url.go | 4 ++-- 6 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 508fe56..1b4b63f 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -13,6 +13,24 @@ type Page struct { Children []*Page } +// NewPage takes a referring Page + a URL and returns a new &Page{} +func NewPage(referrer *Page, rawurl string) (*Page, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.URL.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // create struct and return + return &Page{URL: u}, nil +} + // docrawl begins crawling the site at "url" func docrawl(url string) []*Page { // our result structure @@ -41,13 +59,13 @@ func docrawl(url string) []*Page { parsed[dest.URL.String()] = dest // fetch page - text, err := fetch(dest.URL.String()) + text, err := fetchPage(dest) if err != nil { continue // TODO handle } // parse it - _, links, err := parse(text) + _, links, err := parseLinks(text) if err != nil { continue // TODO handle } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index e3ce469..1641c06 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,7 +58,11 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - body, err := fetch(baseURL + "fetch_test.html") + page, err := NewPage(nil, baseURL+"fetch_test.html") + if err != nil { + t.Error("problem creating New Page struct") + } + body, err := fetchPage(page) if err != nil { t.Fatal(err) } @@ -97,16 +101,30 @@ func TestSimpleMap(t *testing.T) { } } -// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + basepage, err := NewPage(nil, baseURL) + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + + page, err := NewPage(basepage, "about.html") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { - t.Logf("got %q, wanted %q", ft, "text/plain") + + page, err = NewPage(basepage, "assets/image.png") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" { + t.Logf("got %q, wanted %q", page.MediaType, "text/plain") t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 72c5c24..7a4cbdf 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -8,38 +8,45 @@ import ( ) var ( - ErrContentTypeNotFound = errors.New("no Content-Type header found") + errContentTypeNotFound = errors.New("no Content-Type header found") ) -func fetch_filetype(url string) (string, error) { - resp, err := http.Head(url) +// fetchFiletype performs an http HEAD to get the media type, and sets it +// directly in Page.MediaType +func fetchFiletype(page *Page) error { + resp, err := http.Head(page.URL.String()) if err != nil { - return "", err + return err } // pull the content type out of the http header contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found - return "", ErrContentTypeNotFound + return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return err + } + + // set and return + page.MediaType = mediatype + return nil +} + +// fetchPage takes a Page, GETs it, and returns the body as a string +func fetchPage(page *Page) (string, error) { + // figure out the file type + err := fetchFiletype(page) if err != nil { return "", err } - return mediatype, nil -} - -// fetch takes a URL, GETs it, and returns the body as a string -func fetch(url string) (string, error) { - // figure out the file type - //filetype, err := fetch_filetype(url) - // GET the url - resp, err := http.Get(url) + resp, err := http.Get(page.URL.String()) if err != nil { return "", err } diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 13a1010..b951769 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -10,9 +10,9 @@ import ( var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) -// parse takes a string and attempts to parse any html title and all links out of it, +// parseLinks takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found -func parse(s string) (string, []string, error) { +func parseLinks(s string) (string, []string, error) { // find the title, or default to empty title := "" titleMatches := reTitle.FindStringSubmatch(s) diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index 9089d55..64df379 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -12,17 +12,18 @@ func TestSimpleParse(t *testing.T) { - + fooey! +