diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 2331cef..a2c50c7 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net" "net/http" "os" @@ -13,6 +14,7 @@ const ( baseURL = "http://localhost:8765/" ) +// TestMain is used so that we can setup an http server, run tests against it, and tear it down func TestMain(m *testing.M) { // start our simple web server l, err := net.Listen("tcp", ":"+strconv.Itoa(port)) @@ -37,9 +39,59 @@ func TestMain(m *testing.M) { os.Exit(exitcode) } +// TestServerRunning verifies we can get the baseURL from our test harness http server func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { + fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) t.Error(err) } } + +// TestBadRequest verifies we get a 404 for a request to a missing file +func TestBadRequest(t *testing.T) { + resp, err := http.Get(baseURL + "doesnotexist.html") + if resp.StatusCode != 404 || err != nil { + fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Error("request for non existent file didn't fail properly") + } +} + +// TestFetch makes sure we can fetch a file and we get exactly what we expect +func TestFetch(t *testing.T) { + body, err := fetch(baseURL + "fetch_test.html") + if err != nil { + t.Error(err) + } + desired := "
\n" + if body != desired { + fmt.Printf(" Got: '%v'\n", body) + fmt.Printf("Wanted: '%v'\n", desired) + t.Error("fetch succeeded, but content mismatched") + } +} + +// TestParse verifies that we can extract URLs (etc.) from a known document +func TestParse(t *testing.T) { + doc := ` + + + +
+
+
+`
+ matches, err := parse(doc)
+ if err != nil {
+ t.Error(err)
+ }
+ if len(matches) != 2 {
+ t.Error("invalid number of matches in parse")
+ }
+ if matches[0] != "/assets/image.png" {
+ t.Error("match text is invalid")
+ }
+ if matches[1] != "/about.html" {
+ t.Error("match text is invalid")
+ }
+}
diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go
new file mode 100644
index 0000000..bdf1836
--- /dev/null
+++ b/src/docrawler/fetch.go
@@ -0,0 +1,22 @@
+package main
+
+import (
+ "io/ioutil"
+ "net/http"
+)
+
+func fetch(url string) (string, error) {
+ // GET the url
+ resp, err := http.Get(url)
+ if err != nil {
+ return "", err
+ }
+
+ // read the body
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return "", err
+ }
+ resp.Body.Close()
+ return string(body), nil
+}
diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go
new file mode 100644
index 0000000..483dbdb
--- /dev/null
+++ b/src/docrawler/parse.go
@@ -0,0 +1,17 @@
+package main
+
+import (
+ "regexp"
+)
+
+func parse(s string) ([]string, error) {
+ // hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.)
+ re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`)
+
+ // find all of the matches (submatch = include capture groups)
+ results := []string{}
+ for _, match := range re.FindAllStringSubmatch(s, -1) {
+ results = append(results, match[2])
+ }
+ return results, nil
+}
diff --git a/src/docrawler/testsite/fetch_test.html b/src/docrawler/testsite/fetch_test.html
new file mode 100644
index 0000000..30e84fd
--- /dev/null
+++ b/src/docrawler/testsite/fetch_test.html
@@ -0,0 +1 @@
+
diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html
index e69de29..3a41cf9 100644
--- a/src/docrawler/testsite/index.html
+++ b/src/docrawler/testsite/index.html
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+