flow: Closed <feature> 'output_sitemap'.

* the json marshaller cannot deal with circular data structures, and no automatic .String() conversion
* so we copy our crawl data slice into another slice which will be marshaled into json more easily
* made the page slice we use during crawling actually circular (as it should be) rather than putting duplicates in for each page discovered
* test for conversion to output data structure, and hacky cheater test for json output
* removed Base *Page from our Page struct as it's only needed temporarily

Makefile

@@ -3,7 +3,7 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest cover doc run clean env vendor
+.PHONY: build fmt lint vet test autotest cover doc run clean env stats vendor
 
 # configuration
 APPNAME := docrawler
@@ -26,8 +26,8 @@ build: fmt lint vet
 fmt:
 	go fmt ./src/...
 
+# TODO golint is not accepting the "./src/..." for some reason (from within make)
 lint:
-	# TODO golint is not accepting the "./src/..." for some reason (from within make)
 	golint ./src/${APPNAME}/*.go
 
 vet:
@@ -55,6 +55,9 @@ clean:
 env:
 	go env
 
+stats:
+	cloc ./src
+
 # vendor
 # * wipes our vendor directory completely
 # * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)

src/docrawler/docrawler.go

@@ -1,7 +1,5 @@
 package main
 
-import ()
-
 // docrawl begins crawling the site at "url"
 func docrawl(url string) []*Page {
 	// our result structure
@@ -25,11 +23,6 @@ func docrawl(url string) []*Page {
 		}
 		delete(queued, destURL)
 
-		// skip loop if we already did this URL
-		if _, ok := crawled[dest.URL.String()]; ok {
-			continue // TODO handle
-		}
-
 		// mark this page as complete
 		crawled[dest.URL.String()] = dest
 
@@ -40,18 +33,30 @@ func docrawl(url string) []*Page {
 		}
 
 		// parse links
-		_, links, err := parseLinks(text)
+		title, links, err := parseLinks(text)
 		if err != nil {
 			continue // TODO handle
 		}
+		dest.Title = title
 		for _, l := range links {
-			p, _ := NewPage(dest, l) // TODO check error
-			dest.Children = append(dest.Children, p)
+			p, err := NewPage(dest, l)
+			if err != nil {
+				continue // TODO check error
+			}
 
-			// see if we should queue this new page for crawling
-			_, haveCrawled := crawled[p.URL.String()]
-			_, haveQueued := queued[p.URL.String()]
-			if !haveCrawled && !haveQueued {
+			// see if we already know about this page
+			pCrawled, haveCrawled := crawled[p.URL.String()]
+			pQueued, haveQueued := queued[p.URL.String()]
+			if haveCrawled {
+				// add the previously crawled page to children
+				dest.Children = append(dest.Children, pCrawled)
+			} else if haveQueued {
+				// add the previously queued page to children
+				dest.Children = append(dest.Children, pQueued)
+			} else {
+				// add this new page (never seen) to children
+				dest.Children = append(dest.Children, p)
+				// .. and queue it up for crawling
 				queued[p.URL.String()] = p
 			}
 		}

src/docrawler/docrawler_test.go

@@ -18,13 +18,13 @@ func TestMain(m *testing.M) {
 	// start our simple web server
 	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
 	if err != nil {
-		panic("unable to open port for http server") // TODO remove panic?
+		panic("unable to open port for http server")
 	}
 	http.Handle("/", http.FileServer(http.Dir("./testsite")))
 	go func() {
 		err = http.Serve(l, nil)
 		if err != nil {
-			panic("unable to start http server") // TODO remove panic?
+			panic("unable to start http server")
 		}
 	}()
 
@@ -77,7 +77,6 @@ func TestSimpleFetch(t *testing.T) {
 // TestSimpleMap figures out the site map for the site in baseURL
 func TestSimpleMap(t *testing.T) {
 	pages := docrawl(baseURL)
-	t.Log("pages = ", pages)
 	if len(pages) != 4 {
 		t.Logf("got %v, wanted %v\n", len(pages), 1)
 		t.Fatal("got wrong number of pages")
@@ -133,3 +132,49 @@ func TestHeaderFetching(t *testing.T) {
 		t.Error("problem fetching filetype")
 	}
 }
+
+// TestJsonOutput gets a sitemap and then converts it to json
+func TestJsonOutput(t *testing.T) {
+	pages := docrawl(baseURL)
+	l := sitemapToLocations(pages)
+	if len(l) != 2 {
+		t.Error("sitemapToLocations has the wrong number of locations")
+	}
+	if l[0].URL != baseURL {
+		t.Error("got an incorrect location")
+	}
+	if len(l[0].Links) != 1 {
+		t.Error("location 0 has wrong number of links")
+	}
+	if len(l[0].Assets) != 2 {
+		t.Error("location 0 has wrong number of assets")
+	}
+	if len(l[0].Broken) != 0 {
+		t.Error("location 0 has wrong number of broken urls")
+	}
+	if l[1].URL != baseURL+"about.html" {
+		t.Error("got an incorrect location")
+	}
+	if len(l[1].Links) != 1 {
+		t.Error("location 1 has wrong number of links")
+	}
+	if len(l[1].Assets) != 2 {
+		t.Error("location 1 has wrong number of assets")
+	}
+	if len(l[1].Broken) != 1 {
+		t.Error("location 1 has wrong number of broken urls")
+	}
+
+	// this is a cheater test because the output is from a run of the code being
+	// test itself. but, it's been examined and I think it's right. and, there's not
+	// many other ways to test this without some significant pain
+	cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://doesntexist23492387492837492374982734.com/"]}]`
+	j, err := locationsToJSON(l)
+	if err != nil {
+		t.Error("locationsToJSON failed")
+	}
+	if j != cheaterTest {
+		t.Error("locationsToJSON text doesn't match")
+	}
+	t.Log(j)
+}

src/docrawler/fetch_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedHeaderFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
+	}
+}
+
+// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if _, err := fetchPage(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
+	}
+}

src/docrawler/output.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"encoding/json"
+)
+
+// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc.
+type Location struct {
+	URL    string
+	Title  string
+	Links  []string
+	Assets []string
+	Broken []string
+}
+
+func sitemapToLocations(pages []*Page) []*Location {
+	// build a map of our pages
+	pageMap := make(map[string]*Page)
+	for _, p := range pages {
+		pageMap[p.URL.String()] = p
+	}
+
+	// build a slice of locations (one per page)
+	var locations []*Location
+	for _, p := range pages {
+		if p.MediaType == "text/html" {
+			// create a location for this page
+			l := &Location{URL: p.URL.String(), Title: p.Title}
+
+			// add its children
+			for _, c := range p.Children {
+				// look up this child's media type from the root list of pages
+				//mediaType := pageMap[c.URL.String()].MediaType
+				if c.MediaType == "text/html" {
+					l.Links = append(l.Links, c.URL.String())
+				} else if c.MediaType == "" {
+					l.Broken = append(l.Broken, c.URL.String())
+				} else {
+					l.Assets = append(l.Assets, c.URL.String())
+				}
+			}
+
+			// and add this location to our slice
+			locations = append(locations, l)
+		}
+	}
+
+	return locations
+}
+
+func locationsToJSON(locations []*Location) (string, error) {
+	b, err := json.Marshal(locations)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}

src/docrawler/page.go

@@ -7,7 +7,6 @@ import (
 // Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
 type Page struct {
 	URL       *url.URL
-	Base      *Page
 	Title     string
 	MediaType string
 	Children  []*Page

src/docrawler/page_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestBadNewPage tests NewPage to see if we are getting expected failures
+func TestBadNewPage(t *testing.T) {
+	_, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	if err == nil {
+		t.Error("tried creating a NewPage with an invalid URL but didn't get an error")
+	}
+}

src/docrawler/testsite/about.html

@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>About Test</title>
+		<script src="scripts/blah.js"/>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/">fooey!</a>
+		<a href="http://doesntexist23492387492837492374982734.com/">bad one</a>
+	</body>
+</html>

src/docrawler/testsite/index.html

@@ -1,5 +1,6 @@
 <html>
 	<head>
+		<title>Home</title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>

src/docrawler/url_test.go

@@ -52,3 +52,19 @@ func TestURLResolution(t *testing.T) {
 	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
 	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
 }
+
+// TestBadResolve tries to resolve bogus URLs to be sure we're getting errors
+/* TODO not sure how to make url.Parse fail in these cases!!!
+func TestBadResolve(t* testing.T) {
+	u, err := resolveURL("http://a.com/", "h%20t\tp://doesnte%20xist23492387492837492374982734.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+	u, err = resolveURL("h%20t\tp://doesnte%20xist23492387492837492374982734.com/", "http://a.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+}
+*/