diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 1b445d6..5f8e2dd 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -39,13 +39,15 @@ func docrawl(url string) []*Page { // fetch page text, err := fetchPage(dest) if err != nil { - continue // TODO handle + dest.Broken = true + continue } // parse links title, links, err := parseLinks(text) if err != nil { - continue // TODO handle + dest.Broken = true + continue } dest.Title = title for _, l := range links { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 5606863..93660c6 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -177,8 +177,8 @@ func TestHeaderFetching(t *testing.T) { if err != nil { t.Error("problem creating New Page struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" { - t.Logf("got %q, wanted %q", page.MediaType, "text/plain") + if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { + t.Logf("got %q, wanted %q", page.MediaType, "image/png") t.Error("problem fetching filetype") } } @@ -196,14 +196,18 @@ func TestJsonOutput(t *testing.T) { if len(l[0].Links) != 1 { t.Error("location 0 has wrong number of links") } - if len(l[0].Assets) != 3 { + if len(l[0].Assets) != 2 { t.Logf("got %v", len(l[0].Assets)) t.Error("location 0 has wrong number of assets") } - if len(l[0].Broken) != 0 { + if len(l[0].Broken) != 1 { t.Logf("got %v", len(l[0].Broken)) t.Error("location 0 has wrong number of broken urls") } + if len(l[0].Remote) != 0 { + t.Logf("got %v", len(l[1].Remote)) + t.Error("location 0 has wrong number of remote urls") + } if l[1].URL != baseURL+"about.html" { t.Error("got an incorrect location") } @@ -217,11 +221,15 @@ func TestJsonOutput(t *testing.T) { t.Logf("got %v", len(l[1].Broken)) t.Error("location 1 has wrong number of broken urls") } + if len(l[1].Remote) != 1 { + t.Logf("got %v", len(l[1].Remote)) + t.Error("location 1 has wrong number of remote urls") + } // this is a cheater test because the output is from a run of the code being // test itself. but, it's been examined and I think it's right. and, there's not // many other ways to test this without some significant pain - cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js","http://localhost:8765/zzzbroken.html"],"Broken":null,"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]` + cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]` j, err := locationsToJSON(l) if err != nil { t.Error("locationsToJSON failed") @@ -229,4 +237,5 @@ func TestJsonOutput(t *testing.T) { if j != cheaterTest { t.Error("locationsToJSON text doesn't match") } + t.Log(j) } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 7a4cbdf..ebcf9a2 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -9,6 +9,7 @@ import ( var ( errContentTypeNotFound = errors.New("no Content-Type header found") + errFetchError = errors.New("couldn't fetch page") ) // fetchFiletype performs an http HEAD to get the media type, and sets it @@ -19,6 +20,11 @@ func fetchFiletype(page *Page) error { return err } + // check response code + if resp.StatusCode != http.StatusOK { + return errFetchError + } + // pull the content type out of the http header contentType, ok := resp.Header["Content-Type"] if !ok { @@ -45,12 +51,22 @@ func fetchPage(page *Page) (string, error) { return "", err } + // we only want to fetch html + if page.MediaType != "text/html" { + return "", nil // but this isn't an error! + } + // GET the url resp, err := http.Get(page.URL.String()) if err != nil { return "", err } + // check response code + if resp.StatusCode != http.StatusOK { + return "", errFetchError + } + // read the body body, err := ioutil.ReadAll(resp.Body) if err != nil { diff --git a/src/docrawler/output.go b/src/docrawler/output.go index b97730d..87f5ad1 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -33,11 +33,11 @@ func sitemapToLocations(pages []*Page) []*Location { for _, c := range p.Children { // look up this child's media type from the root list of pages //mediaType := pageMap[c.URL.String()].MediaType - if c.Skipped == true { + if c.Skipped { l.Remote = append(l.Remote, c.URL.String()) } else if c.MediaType == "text/html" { l.Links = append(l.Links, c.URL.String()) - } else if c.MediaType == "" { + } else if c.Broken { l.Broken = append(l.Broken, c.URL.String()) } else { l.Assets = append(l.Assets, c.URL.String()) diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 259a1b0..9e8be32 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -10,6 +10,7 @@ type Page struct { Title string MediaType string Skipped bool + Broken bool Children []*Page } @@ -28,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) { } // create struct and return - return &Page{URL: u, Skipped: false}, nil + return &Page{URL: u, Skipped: false, Broken: false}, nil } diff --git a/src/docrawler/testsite/image.png b/src/docrawler/testsite/assets/image.png similarity index 100% rename from src/docrawler/testsite/image.png rename to src/docrawler/testsite/assets/image.png