flow: Closed <feature> 'url_on_command_line'.

* Broken field in Page{} now,
* anything that fails to fetch (with http 200) or parse (should not happen) is considered broken
* tests updated appropriately
* discovered my "image.png" file was in the wrong directory because it showed up as broken. fixed!
* bug fix: only fetch if type=text/html (after HEAD response)

docs/Design.md

@@ -44,7 +44,6 @@ These are features I'd like to include.
 * launch goroutine per fetch
 * when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it
 
-
 ## Wishlist
 
 These are features that sound cool but are probably out of scope.

src/docrawler/docrawler.go

@@ -1,5 +1,10 @@
 package main
 
+import (
+	"fmt"
+	"os"
+)
+
 // docrawl begins crawling the site at "url"
 func docrawl(url string) []*Page {
 	// our result structure
@@ -39,13 +44,15 @@ func docrawl(url string) []*Page {
 		// fetch page
 		text, err := fetchPage(dest)
 		if err != nil {
-			continue // TODO handle
+			dest.Broken = true
+			continue
 		}
 
 		// parse links
 		title, links, err := parseLinks(text)
 		if err != nil {
-			continue // TODO handle
+			dest.Broken = true
+			continue
 		}
 		dest.Title = title
 		for _, l := range links {
@@ -76,4 +83,11 @@ func docrawl(url string) []*Page {
 }
 
 func main() {
+	// crawl each URL on the command line
+	for _, u := range os.Args[1:] {
+		pages := docrawl(u)
+		l := sitemapToLocations(pages)
+		j, _ := locationsToJSON(l)
+		fmt.Println(j)
+	}
 }

src/docrawler/docrawler_test.go

@@ -92,7 +92,7 @@ func TestSimpleMap(t *testing.T) {
 			skippage = p
 		}
 		if p.Skipped {
-			skipped += 1
+			skipped++
 		}
 		children += len(p.Children)
 	}
@@ -177,8 +177,8 @@ func TestHeaderFetching(t *testing.T) {
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" {
-		t.Logf("got %q, wanted %q", page.MediaType, "text/plain")
+	if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" {
+		t.Logf("got %q, wanted %q", page.MediaType, "image/png")
 		t.Error("problem fetching filetype")
 	}
 }
@@ -196,14 +196,18 @@ func TestJsonOutput(t *testing.T) {
 	if len(l[0].Links) != 1 {
 		t.Error("location 0 has wrong number of links")
 	}
-	if len(l[0].Assets) != 3 {
+	if len(l[0].Assets) != 2 {
 		t.Logf("got %v", len(l[0].Assets))
 		t.Error("location 0 has wrong number of assets")
 	}
-	if len(l[0].Broken) != 0 {
+	if len(l[0].Broken) != 1 {
 		t.Logf("got %v", len(l[0].Broken))
 		t.Error("location 0 has wrong number of broken urls")
 	}
+	if len(l[0].Remote) != 0 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 0 has wrong number of remote urls")
+	}
 	if l[1].URL != baseURL+"about.html" {
 		t.Error("got an incorrect location")
 	}
@@ -217,11 +221,15 @@ func TestJsonOutput(t *testing.T) {
 		t.Logf("got %v", len(l[1].Broken))
 		t.Error("location 1 has wrong number of broken urls")
 	}
+	if len(l[1].Remote) != 1 {
+		t.Logf("got %v", len(l[1].Remote))
+		t.Error("location 1 has wrong number of remote urls")
+	}
 
 	// this is a cheater test because the output is from a run of the code being
 	// test itself. but, it's been examined and I think it's right. and, there's not
 	// many other ways to test this without some significant pain
-	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js","http://localhost:8765/zzzbroken.html"],"Broken":null,"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]`
+	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]`
 	j, err := locationsToJSON(l)
 	if err != nil {
 		t.Error("locationsToJSON failed")
@@ -229,4 +237,5 @@ func TestJsonOutput(t *testing.T) {
 	if j != cheaterTest {
 		t.Error("locationsToJSON text doesn't match")
 	}
+	t.Log(j)
 }

src/docrawler/fetch.go

@@ -9,6 +9,7 @@ import (
 
 var (
 	errContentTypeNotFound = errors.New("no Content-Type header found")
+	errFetchError          = errors.New("couldn't fetch page")
 )
 
 // fetchFiletype performs an http HEAD to get the media type, and sets it
@@ -19,6 +20,11 @@ func fetchFiletype(page *Page) error {
 		return err
 	}
 
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return errFetchError
+	}
+
 	// pull the content type out of the http header
 	contentType, ok := resp.Header["Content-Type"]
 	if !ok {
@@ -45,12 +51,22 @@ func fetchPage(page *Page) (string, error) {
 		return "", err
 	}
 
+	// we only want to fetch html
+	if page.MediaType != "text/html" {
+		return "", nil // but this isn't an error!
+	}
+
 	// GET the url
 	resp, err := http.Get(page.URL.String())
 	if err != nil {
 		return "", err
 	}
 
+	// check response code
+	if resp.StatusCode != http.StatusOK {
+		return "", errFetchError
+	}
+
 	// read the body
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {

src/docrawler/output.go

@@ -33,11 +33,11 @@ func sitemapToLocations(pages []*Page) []*Location {
 			for _, c := range p.Children {
 				// look up this child's media type from the root list of pages
 				//mediaType := pageMap[c.URL.String()].MediaType
-				if c.Skipped == true {
+				if c.Skipped {
 					l.Remote = append(l.Remote, c.URL.String())
 				} else if c.MediaType == "text/html" {
 					l.Links = append(l.Links, c.URL.String())
-				} else if c.MediaType == "" {
+				} else if c.Broken {
 					l.Broken = append(l.Broken, c.URL.String())
 				} else {
 					l.Assets = append(l.Assets, c.URL.String())

src/docrawler/page.go

@@ -10,6 +10,7 @@ type Page struct {
 	Title     string
 	MediaType string
 	Skipped   bool
+	Broken    bool
 	Children  []*Page
 }
 
@@ -28,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) {
 	}
 
 	// create struct and return
-	return &Page{URL: u, Skipped: false}, nil
+	return &Page{URL: u, Skipped: false, Broken: false}, nil
 }