diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 2331cef..a2c50c7 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net" "net/http" "os" @@ -13,6 +14,7 @@ const ( baseURL = "http://localhost:8765/" ) +// TestMain is used so that we can setup an http server, run tests against it, and tear it down func TestMain(m *testing.M) { // start our simple web server l, err := net.Listen("tcp", ":"+strconv.Itoa(port)) @@ -37,9 +39,59 @@ func TestMain(m *testing.M) { os.Exit(exitcode) } +// TestServerRunning verifies we can get the baseURL from our test harness http server func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { + fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) t.Error(err) } } + +// TestBadRequest verifies we get a 404 for a request to a missing file +func TestBadRequest(t *testing.T) { + resp, err := http.Get(baseURL + "doesnotexist.html") + if resp.StatusCode != 404 || err != nil { + fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Error("request for non existent file didn't fail properly") + } +} + +// TestFetch makes sure we can fetch a file and we get exactly what we expect +func TestFetch(t *testing.T) { + body, err := fetch(baseURL + "fetch_test.html") + if err != nil { + t.Error(err) + } + desired := "\n" + if body != desired { + fmt.Printf(" Got: '%v'\n", body) + fmt.Printf("Wanted: '%v'\n", desired) + t.Error("fetch succeeded, but content mismatched") + } +} + +// TestParse verifies that we can extract URLs (etc.) from a known document +func TestParse(t *testing.T) { + doc := ` + + + + + + +` + matches, err := parse(doc) + if err != nil { + t.Error(err) + } + if len(matches) != 2 { + t.Error("invalid number of matches in parse") + } + if matches[0] != "/assets/image.png" { + t.Error("match text is invalid") + } + if matches[1] != "/about.html" { + t.Error("match text is invalid") + } +} diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go new file mode 100644 index 0000000..bdf1836 --- /dev/null +++ b/src/docrawler/fetch.go @@ -0,0 +1,22 @@ +package main + +import ( + "io/ioutil" + "net/http" +) + +func fetch(url string) (string, error) { + // GET the url + resp, err := http.Get(url) + if err != nil { + return "", err + } + + // read the body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + resp.Body.Close() + return string(body), nil +} diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go new file mode 100644 index 0000000..483dbdb --- /dev/null +++ b/src/docrawler/parse.go @@ -0,0 +1,17 @@ +package main + +import ( + "regexp" +) + +func parse(s string) ([]string, error) { + // hacky RE to capture any kind of URL link in html ('a', 'img', 'script', 'video', etc.) + re := regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) + + // find all of the matches (submatch = include capture groups) + results := []string{} + for _, match := range re.FindAllStringSubmatch(s, -1) { + results = append(results, match[2]) + } + return results, nil +} diff --git a/src/docrawler/testsite/fetch_test.html b/src/docrawler/testsite/fetch_test.html new file mode 100644 index 0000000..30e84fd --- /dev/null +++ b/src/docrawler/testsite/fetch_test.html @@ -0,0 +1 @@ + diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html index e69de29..3a41cf9 100644 --- a/src/docrawler/testsite/index.html +++ b/src/docrawler/testsite/index.html @@ -0,0 +1,8 @@ + + + + + + + +