flow: Closed <feature> 'fix_broken_links'.

* Broken field in Page{} now,
* anything that fails to fetch (with http 200) or parse (should not happen) is considered broken
* tests updated appropriately
* discovered my "image.png" file was in the wrong directory because it showed up as broken. fixed!
* bug fix: only fetch if type=text/html (after HEAD response)

src/docrawler/docrawler.go

@@ -26,16 +26,28 @@ func docrawl(url string) []*Page {
 		// mark this page as complete
 		crawled[dest.URL.String()] = dest
 
+		// add page to list
+		sitemap = append(sitemap, dest)
+
+		// make sure this URL has the same hostname as our first page
+		if dest.URL.Host != homepage.URL.Host {
+			// skip URLs associated with other Hosts
+			dest.Skipped = true
+			continue
+		}
+
 		// fetch page
 		text, err := fetchPage(dest)
 		if err != nil {
-			continue // TODO handle
+			dest.Broken = true
+			continue
 		}
 
 		// parse links
 		title, links, err := parseLinks(text)
 		if err != nil {
-			continue // TODO handle
+			dest.Broken = true
+			continue
 		}
 		dest.Title = title
 		for _, l := range links {
@@ -60,9 +72,6 @@ func docrawl(url string) []*Page {
 				queued[p.URL.String()] = p
 			}
 		}
-
-		// add page to list
-		sitemap = append(sitemap, dest)
 	}
 
 	return sitemap

src/docrawler/docrawler_test.go

@@ -77,32 +77,82 @@ func TestSimpleFetch(t *testing.T) {
 // TestSimpleMap figures out the site map for the site in baseURL
 func TestSimpleMap(t *testing.T) {
 	pages := docrawl(baseURL)
-	if len(pages) != 4 {
-		t.Logf("got %v, wanted %v\n", len(pages), 1)
+
+	// because our crawl is non-deterministic, we have to do a complete
+	// cycle through every page, counting stuff, finding specific pages
+	// so that we can then verify certain aspects of each page
+	skipped, children := 0, 0
+	var homepage, aboutpage, skippage *Page
+	for _, p := range pages {
+		if p.Title == "Home" {
+			homepage = p
+		} else if p.Title == "About Test" {
+			aboutpage = p
+		} else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" {
+			skippage = p
+		}
+		if p.Skipped {
+			skipped++
+		}
+		children += len(p.Children)
+	}
+
+	// check page counts
+	if len(pages) != 6 {
+		t.Logf("got %v, wanted %v\n", len(pages), 6)
 		t.Fatal("got wrong number of pages")
 	}
-	if pages[0].URL.String() != baseURL {
+	if children != 8 {
+		t.Logf("got %v, wanted %v\n", children, 8)
+		t.Fatal("got wrong number of total children")
+	}
+	if skipped != 1 {
+		t.Logf("got %v, wanted %v\n", skipped, 1)
+		t.Fatal("got wrong number of total skipped")
+	}
+
+	// check first page URL
+	if homepage.URL.String() != baseURL {
 		t.Error("page URL is invalid")
 	}
-	if len(pages[0].Children) != 3 {
-		t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1)
+
+	// specific page number of children
+	if len(homepage.Children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(homepage.Children), 4)
 		t.Fatal("got wrong number of links")
 	}
-	if pages[0].Children[0].URL.String() != baseURL+"about.html" {
-		t.Logf("   Got: %q\n", pages[0].Children[0].URL.String())
-		t.Logf("Wanted: %q\n", baseURL+"about.html")
-		t.Error("link name is incorrect")
+	if len(aboutpage.Children) != 4 {
+		t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4)
+		t.Fatal("got wrong number of links")
 	}
-	if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" {
-		t.Logf("   Got: %q\n", pages[0].Children[1].URL.String())
-		t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
-		t.Error("link name is incorrect")
+
+	// make sure remote pages were skipped
+	if homepage.Skipped != false {
+		t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String())
+		t.Error("child was skipped")
 	}
-	if pages[0].Children[2].URL.String() != baseURL+"scripts/blah.js" {
-		t.Logf("   Got: %q\n", pages[0].Children[2].URL.String())
-		t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
-		t.Error("link name is incorrect")
+	if skippage.Skipped != true {
+		t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String())
+		t.Error("child wasn't skipped")
 	}
+
+	// verify page content
+	// TODO find a better way to do this
+	//if homepage.Children[0].URL.String() != baseURL+"about.html" {
+	//t.Logf("   Got: %q\n", homepage.Children[0].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"about.html")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.Children[1].URL.String() != baseURL+"assets/image.png" {
+	//t.Logf("   Got: %q\n", homepage.Children[1].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
+	//t.Error("link name is incorrect")
+	//}
+	//if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" {
+	//t.Logf("   Got: %q\n", homepage.Children[3].URL.String())
+	//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+	//t.Error("link name is incorrect")
+	//}
 }
 
 // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
@@ -127,8 +177,8 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" {
-		t.Logf("got %q, wanted %q", page.MediaType, "text/plain")
+	if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" {
+		t.Logf("got %q, wanted %q", page.MediaType, "image/png")
 		t.Error("problem fetching filetype")
 	}
 }
@@ -147,11 +197,17 @@ func TestJsonOutput(t *testing.T) {
 		t.Error("location 0 has wrong number of links")
 	}
 	if len(l[0].Assets) != 2 {
+		t.Logf("got %v", len(l[0].Assets))
 		t.Error("location 0 has wrong number of assets")
 	}
-	if len(l[0].Broken) != 0 {
+	if len(l[0].Broken) != 1 {
+		t.Logf("got %v", len(l[0].Broken))
 		t.Error("location 0 has wrong number of broken urls")
 	}
+	if len(l[0].Remote) != 0 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 0 has wrong number of remote urls")
+	}
 	if l[1].URL != baseURL+"about.html" {
 		t.Error("got an incorrect location")
 	}
@@ -161,14 +217,19 @@ func TestJsonOutput(t *testing.T) {
 	if len(l[1].Assets) != 2 {
 		t.Error("location 1 has wrong number of assets")
 	}
-	if len(l[1].Broken) != 1 {
+	if len(l[1].Broken) != 0 {
+		t.Logf("got %v", len(l[1].Broken))
 		t.Error("location 1 has wrong number of broken urls")
 	}
+	if len(l[1].Remote) != 1 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 1 has wrong number of remote urls")
+	}
 
 	// this is a cheater test because the output is from a run of the code being
 	// test itself. but, it's been examined and I think it's right. and, there's not
 	// many other ways to test this without some significant pain
-	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://doesntexist23492387492837492374982734.com/"]}]`
+	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]`
 	j, err := locationsToJSON(l)
 	if err != nil {
 		t.Error("locationsToJSON failed")

src/docrawler/fetch.go

@@ -9,6 +9,7 @@ import (
 
 var (
 	errContentTypeNotFound = errors.New("no Content-Type header found")
+	errFetchError          = errors.New("couldn't fetch page")
 )
 
 // fetchFiletype performs an http HEAD to get the media type, and sets it
@@ -19,6 +20,11 @@ func fetchFiletype(page *Page) error {
 		return err
 	}
 
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return errFetchError
+	}
+
 	// pull the content type out of the http header
 	contentType, ok := resp.Header["Content-Type"]
 	if !ok {
@@ -45,12 +51,22 @@ func fetchPage(page *Page) (string, error) {
 		return "", err
 	}
 
+	// we only want to fetch html
+	if page.MediaType != "text/html" {
+		return "", nil // but this isn't an error!
+	}
+
 	// GET the url
 	resp, err := http.Get(page.URL.String())
 	if err != nil {
 		return "", err
 	}
 
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return "", errFetchError
+	}
+
 	// read the body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {

src/docrawler/output.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"sort"
 )
 
 // Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc.
@@ -11,6 +12,7 @@ type Location struct {
 	Links  []string
 	Assets []string
 	Broken []string
+	Remote []string
 }
 
 func sitemapToLocations(pages []*Page) []*Location {
@@ -31,15 +33,23 @@ func sitemapToLocations(pages []*Page) []*Location {
 			for _, c := range p.Children {
 				// look up this child's media type from the root list of pages
 				//mediaType := pageMap[c.URL.String()].MediaType
-				if c.MediaType == "text/html" {
+				if c.Skipped {
+					l.Remote = append(l.Remote, c.URL.String())
+				} else if c.MediaType == "text/html" {
 					l.Links = append(l.Links, c.URL.String())
-				} else if c.MediaType == "" {
+				} else if c.Broken {
 					l.Broken = append(l.Broken, c.URL.String())
 				} else {
 					l.Assets = append(l.Assets, c.URL.String())
 				}
 			}
 
+			// now sort the children slices
+			sort.Strings(l.Remote)
+			sort.Strings(l.Links)
+			sort.Strings(l.Broken)
+			sort.Strings(l.Assets)
+
 			// and add this location to our slice
 			locations = append(locations, l)
 		}

src/docrawler/page.go

@@ -9,6 +9,8 @@ type Page struct {
 	URL       *url.URL
 	Title     string
 	MediaType string
+	Skipped   bool
+	Broken    bool
 	Children  []*Page
 }
 
@@ -27,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) {
 	}
 
 	// create struct and return
-	return &Page{URL: u}, nil
+	return &Page{URL: u, Skipped: false, Broken: false}, nil
 }

src/docrawler/parse.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"regexp"
-	"sort"
 	"strings"
 )
 
@@ -11,7 +10,7 @@ var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO m
 var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
 // parseLinks takes a string and attempts to parse any html title and all links out of it,
-// and returns a sorted slice of the captures found
+// and returns a slice of the captures found
 func parseLinks(s string) (string, []string, error) {
 	// find the title, or default to empty
 	title := ""
@@ -26,6 +25,5 @@ func parseLinks(s string) (string, []string, error) {
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
-	sort.Strings(results)
 	return title, results, nil
 }

src/docrawler/parse_test.go

@@ -26,10 +26,10 @@ func TestSimpleParse(t *testing.T) {
 	if len(matches) != 3 {
 		t.Error("invalid number of matches in parse")
 	}
-	if matches[0] != "/about.html" {
+	if matches[0] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
-	if matches[1] != "/assets/image.png" {
+	if matches[1] != "/about.html" {
 		t.Error("match text is invalid")
 	}
 	if matches[2] != "scripts/blah.js" {

src/docrawler/testsite/index.html

@@ -6,5 +6,6 @@
 		<img src="/assets/image.png"/>
 		<a href="/about.html">fooey!</a>
 		<script src="scripts/blah.js"/>
+		<a href="/zzzbroken.html">oops!</a>
 	</body>
 </html>