misc. minor cleanup

* moved $/testfilter.sh to $/util/testfilter.sh (updated Makefile accordingly)
* updated idea in Design.md
* renamed TestParse/TestFetch to TestSimpleParse/TestSimpleFetch.. more tests coming
* parse now sorts its results. test updated.
* moved regexp compilation to init instead of every time parse() is called
* misc comments and cleanup

Makefile

@@ -31,11 +31,11 @@ vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./testfilter.sh
+	go test ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \
-	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./testfilter.sh'
+	xargs -n1 -I{} ${SHELL} -c 'echo ; date ; echo ----------------------------; go test ./src/... | ./util/testfilter.sh'
 
 run: build
 	./bin/${APPNAME}

docs/Design.md

@@ -39,6 +39,7 @@ These are features I'd like to include.
 
 * use channels to form a pipeline
 * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines)
+* pipeline may allows us to scale concurrency at the bottlenecks
 * could use Interfaces for things like Parser, Getter, but I think best avoided until needed
 * launch goroutine per fetch
 * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it

src/docrawler/docrawler_test.go

@@ -57,8 +57,8 @@ func TestBadRequest(t *testing.T) {
 	}
 }
 
-// TestFetch makes sure we can fetch a file and we get exactly what we expect
+// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
-func TestFetch(t *testing.T) {
+func TestSimpleFetch(t *testing.T) {
 	body, err := fetch(baseURL + "fetch_test.html")
 	if err != nil {
 		t.Error(err)
@@ -71,8 +71,8 @@ func TestFetch(t *testing.T) {
 	}
 }
 
-// TestParse verifies that we can extract URLs (etc.) from a known document
+// TestSimpleParse verifies that we can extract URLs (etc.) from a known document
-func TestParse(t *testing.T) {
+func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
 	</head>
@@ -88,10 +88,10 @@ func TestParse(t *testing.T) {
 	if len(matches) != 2 {
 		t.Error("invalid number of matches in parse")
 	}
-	if matches[0] != "/assets/image.png" {
+	if matches[0] != "/about.html" {
 		t.Error("match text is invalid")
 	}
-	if matches[1] != "/about.html" {
+	if matches[1] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
 }

src/docrawler/fetch.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 )
 
+// fetch takes a URL, GETs it, and returns the body as a string
 func fetch(url string) (string, error) {
 	// GET the url
 	resp, err := http.Get(url)
@@ -17,6 +18,8 @@ func fetch(url string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
+	// success! close the body, return it as a string
 	resp.Body.Close()
 	return string(body), nil
 }

src/docrawler/parse.go

@@ -2,16 +2,20 @@ package main
 
 import (
 	"regexp"
+	"sort"
 )
 
-func parse(s string) ([]string, error) {
+// a regexp which captures src/href/xhref="..." from text
-	// hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.)
+var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice
-	re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`)
 
+// parse takes a string and attempts to parse any html links out of it,
+// and returns a sorted slice of the captures found
+func parse(s string) ([]string, error) {
 	// find all of the matches (submatch = include capture groups)
 	results := []string{}
-	for _, match := range re.FindAllStringSubmatch(s, -1) {
+	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
+	sort.Strings(results)
 	return results, nil
 }