diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 1134f4a..508fe56 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,18 +1,66 @@ package main -import () +import ( + "net/url" +) -// Page is a struct which defines a single page, which URLs (links and assets) it contains +// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { - URL string - Assets []string - Links []string + URL *url.URL + Base *Page + Title string + MediaType string + Children []*Page } // docrawl begins crawling the site at "url" -func docrawl(url string) []Page { - var result []Page - return result +func docrawl(url string) []*Page { + // our result structure + var sitemap []*Page + + // what we've already parsed, what we still need to parse + parsed := make(map[string]*Page) + var queued []*Page + + // create first page and add to the queue + u, _ := resolveURL("", url) // TODO check error + queued = append(queued, &Page{URL: u}) + + // main loop + for len(queued) > 0 { + // pop a page off of queued + var dest *Page + dest, queued = queued[len(queued)-1], queued[:len(queued)-1] + + // skip loop if we already did this URL + if _, ok := parsed[dest.URL.String()]; ok { + continue // TODO handle + } + + // mark this page as complete + parsed[dest.URL.String()] = dest + + // fetch page + text, err := fetch(dest.URL.String()) + if err != nil { + continue // TODO handle + } + + // parse it + _, links, err := parse(text) + if err != nil { + continue // TODO handle + } + for _, l := range links { + u, _ := resolveURL(dest.URL.String(), l) // TODO check error + dest.Children = append(dest.Children, &Page{URL: u, Base: dest}) + } + + // add page to list + sitemap = append(sitemap, dest) + } + + return sitemap } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index eacf080..e3ce469 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net" "net/http" "os" @@ -43,7 +42,7 @@ func TestMain(m *testing.M) { func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal(err) } } @@ -52,7 +51,7 @@ func TestServerRunning(t *testing.T) { func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal("request for non existent file didn't fail properly") } } @@ -65,8 +64,8 @@ func TestSimpleFetch(t *testing.T) { } desired := "\n" if body != desired { - fmt.Printf(" Got: '%v'\n", body) - fmt.Printf("Wanted: '%v'\n", desired) + t.Logf(" Got: '%v'\n", body) + t.Logf("Wanted: '%v'\n", desired) t.Error("fetch succeeded, but content mismatched") } } @@ -74,22 +73,40 @@ func TestSimpleFetch(t *testing.T) { // TestSimpleMap figures out the site map for the site in baseURL func TestSimpleMap(t *testing.T) { pages := docrawl(baseURL) + t.Log("pages = ", pages) if len(pages) != 1 { + t.Logf("got %v, wanted %v\n", len(pages), 1) t.Fatal("got wrong number of pages") } - if pages[0].URL != baseURL { + if pages[0].URL.String() != baseURL { t.Error("page URL is invalid") } - if len(pages[0].Assets) != 1 { - t.Fatal("got wrong number of assets") - } - if pages[0].Assets[0] != baseURL+"/assets/image.png" { - t.Error("asset name is incorrect") - } - if len(pages[0].Links) != 1 { + if len(pages[0].Children) != 2 { + t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1) t.Fatal("got wrong number of links") } - if pages[0].Links[0] != baseURL+"/about.html" { + if pages[0].Children[0].URL.String() != baseURL+"about.html" { + t.Logf(" Got: %q\n", pages[0].Children[0].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"about.html") + t.Error("link name is incorrect") + } + if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" { + t.Logf(" Got: %q\n", pages[0].Children[1].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"assets/image.png") t.Error("link name is incorrect") } } + +// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +func TestHeaderFetching(t *testing.T) { + if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { + t.Logf("got %q, wanted %q", ft, "text/plain") + t.Error("problem fetching filetype") + } +} diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 4142836..72c5c24 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -1,12 +1,43 @@ package main import ( + "errors" "io/ioutil" + "mime" "net/http" ) +var ( + ErrContentTypeNotFound = errors.New("no Content-Type header found") +) + +func fetch_filetype(url string) (string, error) { + resp, err := http.Head(url) + if err != nil { + return "", err + } + + // pull the content type out of the http header + contentType, ok := resp.Header["Content-Type"] + if !ok { + // no Content-Type header found + return "", ErrContentTypeNotFound + } + + // parse mime type + mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return "", err + } + + return mediatype, nil +} + // fetch takes a URL, GETs it, and returns the body as a string func fetch(url string) (string, error) { + // figure out the file type + //filetype, err := fetch_filetype(url) + // GET the url resp, err := http.Get(url) if err != nil { diff --git a/src/docrawler/testsite/about.html b/src/docrawler/testsite/about.html new file mode 100644 index 0000000..e69de29 diff --git a/src/docrawler/testsite/image.png b/src/docrawler/testsite/image.png new file mode 100644 index 0000000..db549f4 Binary files /dev/null and b/src/docrawler/testsite/image.png differ diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 5381b64..38379df 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/url" "testing" ) @@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) { } cleanURL(u) if u.Host != "somehost.com:8765" { - fmt.Printf(" Got: %q\n", u.Host) - fmt.Printf("Wanted: %q\n", "somehost.com:8765") + t.Logf(" Got: %q\n", u.Host) + t.Logf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { @@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) { t.Error("resolveURL failed (probably parsing)") } if u.String() != wanted { - fmt.Printf(" Got: %q\n", u) - fmt.Printf("Wanted: %q\n", wanted) + t.Logf(" Got: %q\n", u) + t.Logf("Wanted: %q\n", wanted) t.Error("url resolution with base URL failed") } } @@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) { doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") + doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html") }