diff --git a/Makefile b/Makefile index 9c4855c..e4cdaac 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,11 @@ vet: go vet ./src/... test: - go test ./src/... | ./testfilter.sh + go test ./src/... | ./util/testfilter.sh autotest: fswatch -or ./src | \ - xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./testfilter.sh' + xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh' run: build ./bin/${APPNAME} diff --git a/docs/Design.md b/docs/Design.md index 183f389..d0e11ee 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -39,6 +39,7 @@ These are features I'd like to include. * use channels to form a pipeline * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines) +* pipeline may allows us to scale concurrency at the bottlenecks * could use Interfaces for things like Parser, Getter, but I think best avoided until needed * launch goroutine per fetch * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index a2c50c7..10dae64 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -57,8 +57,8 @@ func TestBadRequest(t *testing.T) { } } -// TestFetch makes sure we can fetch a file and we get exactly what we expect -func TestFetch(t *testing.T) { +// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect +func TestSimpleFetch(t *testing.T) { body, err := fetch(baseURL + "fetch_test.html") if err != nil { t.Error(err) @@ -71,8 +71,8 @@ func TestFetch(t *testing.T) { } } -// TestParse verifies that we can extract URLs (etc.) from a known document -func TestParse(t *testing.T) { +// TestSimpleParse verifies that we can extract URLs (etc.) from a known document +func TestSimpleParse(t *testing.T) { doc := ` @@ -88,10 +88,10 @@ func TestParse(t *testing.T) { if len(matches) != 2 { t.Error("invalid number of matches in parse") } - if matches[0] != "/assets/image.png" { + if matches[0] != "/about.html" { t.Error("match text is invalid") } - if matches[1] != "/about.html" { + if matches[1] != "/assets/image.png" { t.Error("match text is invalid") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index bdf1836..4142836 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -5,6 +5,7 @@ import ( "net/http" ) +// fetch takes a URL, GETs it, and returns the body as a string func fetch(url string) (string, error) { // GET the url resp, err := http.Get(url) @@ -17,6 +18,8 @@ func fetch(url string) (string, error) { if err != nil { return "", err } + + // success! close the body, return it as a string resp.Body.Close() return string(body), nil } diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 483dbdb..30e678a 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -2,16 +2,20 @@ package main import ( "regexp" + "sort" ) -func parse(s string) ([]string, error) { - // hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.) - re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) +// a regexp which captures src/href/xhref="..." from text +var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice +// parse takes a string and attempts to parse any html links out of it, +// and returns a sorted slice of the captures found +func parse(s string) ([]string, error) { // find all of the matches (submatch = include capture groups) results := []string{} - for _, match := range re.FindAllStringSubmatch(s, -1) { + for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } + sort.Strings(results) return results, nil } diff --git a/testfilter.sh b/util/testfilter.sh similarity index 100% rename from testfilter.sh rename to util/testfilter.sh