add tests for very simple fetching and parsing

* added test for fetching a known file and verifying its content matches
* add test for parsing a known string and verify URLs are extracted
* add test for a request which doesn't exist and verify 404 is received (canary test)
* add comments throughout test (for each test function)
* wrote (stupidly) simple fetcher which takes a URL, GETs it, and returns the body and error
* wrote (stupidly) simple parser which takes a string, parses out (src|href|xhref)="...", and returns a slice of the captured links
* added needed html files

src/docrawler/docrawler_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -13,6 +14,7 @@ const (
 	baseURL = "http://localhost:8765/"
 )
 
+// TestMain is used so that we can setup an http server, run tests against it, and tear it down
 func TestMain(m *testing.M) {
 	// start our simple web server
 	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
@@ -37,9 +39,59 @@ func TestMain(m *testing.M) {
 	os.Exit(exitcode)
 }
 
+// TestServerRunning verifies we can get the baseURL from our test harness http server
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
+		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
 		t.Error(err)
 	}
 }
+
+// TestBadRequest verifies we get a 404 for a request to a missing file
+func TestBadRequest(t *testing.T) {
+	resp, err := http.Get(baseURL + "doesnotexist.html")
+	if resp.StatusCode != 404 || err != nil {
+		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Error("request for non existent file didn't fail properly")
+	}
+}
+
+// TestFetch makes sure we can fetch a file and we get exactly what we expect
+func TestFetch(t *testing.T) {
+	body, err := fetch(baseURL + "fetch_test.html")
+	if err != nil {
+		t.Error(err)
+	}
+	desired := "<html><head></head><body></body></html>\n"
+	if body != desired {
+		fmt.Printf("   Got: '%v'\n", body)
+		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Error("fetch succeeded, but content mismatched")
+	}
+}
+
+// TestParse verifies that we can extract URLs (etc.) from a known document
+func TestParse(t *testing.T) {
+	doc := `<html>
+	<head>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/about.html">
+	</body>
+</html>`
+	matches, err := parse(doc)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(matches) != 2 {
+		t.Error("invalid number of matches in parse")
+	}
+	if matches[0] != "/assets/image.png" {
+		t.Error("match text is invalid")
+	}
+	if matches[1] != "/about.html" {
+		t.Error("match text is invalid")
+	}
+}

src/docrawler/fetch.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+)
+
+func fetch(url string) (string, error) {
+	// GET the url
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+
+	// read the body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	resp.Body.Close()
+	return string(body), nil
+}

src/docrawler/parse.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"regexp"
+)
+
+func parse(s string) ([]string, error) {
+	// hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.)
+	re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`)
+
+	// find all of the matches (submatch = include capture groups)
+	results := []string{}
+	for _, match := range re.FindAllStringSubmatch(s, -1) {
+		results = append(results, match[2])
+	}
+	return results, nil
+}

src/docrawler/testsite/fetch_test.html

@@ -0,0 +1 @@
+<html><head></head><body></body></html>

src/docrawler/testsite/index.html

@@ -0,0 +1,8 @@
+<html>
+	<head>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/about.html">
+	</body>
+</html>