minimally functional crawler - crawls single page, finds links, stores in data structure

* refined Page struct. now need to refactor everything else to use this
* wrote docrawl() function. NOT intended to be anything like the final design. just sketching it out to get a better idea of needs, design, etc.
* replaced fmt.Print* with t.Log* in tests (oops!)
* added a header fetcher function to get the file type (since we only want to parse html (with tests)
* added small test for the URL resolver where the base URL is empty
* added placeholder files in testsite/ for about.html and image.png

src/docrawler/docrawler.go

@@ -1,18 +1,66 @@
 package main
 
-import ()
+import (
+	"net/url"
+)
 
-// Page is a struct which defines a single page, which URLs (links and assets) it contains
+// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
 type Page struct {
-	URL    string
+	URL       *url.URL
-	Assets []string
+	Base      *Page
-	Links  []string
+	Title     string
+	MediaType string
+	Children  []*Page
 }
 
 // docrawl begins crawling the site at "url"
-func docrawl(url string) []Page {
+func docrawl(url string) []*Page {
-	var result []Page
+	// our result structure
-	return result
+	var sitemap []*Page
+
+	// what we've already parsed, what we still need to parse
+	parsed := make(map[string]*Page)
+	var queued []*Page
+
+	// create first page and add to the queue
+	u, _ := resolveURL("", url) // TODO check error
+	queued = append(queued, &Page{URL: u})
+
+	// main loop
+	for len(queued) > 0 {
+		// pop a page off of queued
+		var dest *Page
+		dest, queued = queued[len(queued)-1], queued[:len(queued)-1]
+
+		// skip loop if we already did this URL
+		if _, ok := parsed[dest.URL.String()]; ok {
+			continue // TODO handle
+		}
+
+		// mark this page as complete
+		parsed[dest.URL.String()] = dest
+
+		// fetch page
+		text, err := fetch(dest.URL.String())
+		if err != nil {
+			continue // TODO handle
+		}
+
+		// parse it
+		_, links, err := parse(text)
+		if err != nil {
+			continue // TODO handle
+		}
+		for _, l := range links {
+			u, _ := resolveURL(dest.URL.String(), l) // TODO check error
+			dest.Children = append(dest.Children, &Page{URL: u, Base: dest})
+		}
+
+		// add page to list
+		sitemap = append(sitemap, dest)
+	}
+
+	return sitemap
 }
 
 func main() {

src/docrawler/docrawler_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -43,7 +42,7 @@ func TestMain(m *testing.M) {
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
 		t.Fatal(err)
 	}
 }
@@ -52,7 +51,7 @@ func TestServerRunning(t *testing.T) {
 func TestBadRequest(t *testing.T) {
 	resp, err := http.Get(baseURL + "doesnotexist.html")
 	if resp.StatusCode != 404 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
 		t.Fatal("request for non existent file didn't fail properly")
 	}
 }
@@ -65,8 +64,8 @@ func TestSimpleFetch(t *testing.T) {
 	}
 	desired := "<html><head></head><body></body></html>\n"
 	if body != desired {
-		fmt.Printf("   Got: '%v'\n", body)
+		t.Logf("   Got: '%v'\n", body)
-		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Logf("Wanted: '%v'\n", desired)
 		t.Error("fetch succeeded, but content mismatched")
 	}
 }
@@ -74,22 +73,40 @@ func TestSimpleFetch(t *testing.T) {
 // TestSimpleMap figures out the site map for the site in baseURL
 func TestSimpleMap(t *testing.T) {
 	pages := docrawl(baseURL)
+	t.Log("pages = ", pages)
 	if len(pages) != 1 {
+		t.Logf("got %v, wanted %v\n", len(pages), 1)
 		t.Fatal("got wrong number of pages")
 	}
-	if pages[0].URL != baseURL {
+	if pages[0].URL.String() != baseURL {
 		t.Error("page URL is invalid")
 	}
-	if len(pages[0].Assets) != 1 {
+	if len(pages[0].Children) != 2 {
-		t.Fatal("got wrong number of assets")
+		t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1)
-	}
-	if pages[0].Assets[0] != baseURL+"/assets/image.png" {
-		t.Error("asset name is incorrect")
-	}
-	if len(pages[0].Links) != 1 {
 		t.Fatal("got wrong number of links")
 	}
-	if pages[0].Links[0] != baseURL+"/about.html" {
+	if pages[0].Children[0].URL.String() != baseURL+"about.html" {
+		t.Logf("   Got: %q\n", pages[0].Children[0].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"about.html")
+		t.Error("link name is incorrect")
+	}
+	if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" {
+		t.Logf("   Got: %q\n", pages[0].Children[1].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
 		t.Error("link name is incorrect")
 	}
 }
+
+// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results
+func TestHeaderFetching(t *testing.T) {
+	if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" {
+		t.Error("problem fetching filetype")
+	}
+	if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" {
+		t.Error("problem fetching filetype")
+	}
+	if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" {
+		t.Logf("got %q, wanted %q", ft, "text/plain")
+		t.Error("problem fetching filetype")
+	}
+}

src/docrawler/fetch.go

@@ -1,12 +1,43 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 )
 
+var (
+	ErrContentTypeNotFound = errors.New("no Content-Type header found")
+)
+
+func fetch_filetype(url string) (string, error) {
+	resp, err := http.Head(url)
+	if err != nil {
+		return "", err
+	}
+
+	// pull the content type out of the http header
+	contentType, ok := resp.Header["Content-Type"]
+	if !ok {
+		// no Content-Type header found
+		return "", ErrContentTypeNotFound
+	}
+
+	// parse mime type
+	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		return "", err
+	}
+
+	return mediatype, nil
+}
+
 // fetch takes a URL, GETs it, and returns the body as a string
 func fetch(url string) (string, error) {
+	// figure out the file type
+	//filetype, err := fetch_filetype(url)
+
 	// GET the url
 	resp, err := http.Get(url)
 	if err != nil {

src/docrawler/url_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net/url"
 	"testing"
 )
@@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) {
 	}
 	cleanURL(u)
 	if u.Host != "somehost.com:8765" {
-		fmt.Printf("   Got: %q\n", u.Host)
+		t.Logf("   Got: %q\n", u.Host)
-		fmt.Printf("Wanted: %q\n", "somehost.com:8765")
+		t.Logf("Wanted: %q\n", "somehost.com:8765")
 		t.Error("host extraction failed")
 	}
 	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
@@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) {
 		t.Error("resolveURL failed (probably parsing)")
 	}
 	if u.String() != wanted {
-		fmt.Printf("   Got: %q\n", u)
+		t.Logf("   Got: %q\n", u)
-		fmt.Printf("Wanted: %q\n", wanted)
+		t.Logf("Wanted: %q\n", wanted)
 		t.Error("url resolution with base URL failed")
 	}
 }
@@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) {
 	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
 	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
 	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
 }