now crawling multiple pages!

* moved Page{} and NewPage() into page.go
* queued and crawled are both maps now. used to track which pages we already know about
* tests updated to match the multipage crawling
* added another file to the testsite (scripts/blah.js)

src/docrawler/docrawler.go

@@ -1,62 +1,38 @@
 package main
 
 import (
-	"net/url"
 )
 
-// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
-type Page struct {
-	URL       *url.URL
-	Base      *Page
-	Title     string
-	MediaType string
-	Children  []*Page
-}
-
-// NewPage takes a referring Page + a URL and returns a new &Page{}
-func NewPage(referrer *Page, rawurl string) (*Page, error) {
-	// determine the base URL so we can resulve this into a full URL
-	baseURL := ""
-	if referrer != nil {
-		baseURL = referrer.URL.String()
-	}
-
-	// resolve URL into a full absolute URL (non-relative)
-	u, err := resolveURL(baseURL, rawurl)
-	if err != nil {
-		return nil, err
-	}
-
-	// create struct and return
-	return &Page{URL: u}, nil
-}
-
 // docrawl begins crawling the site at "url"
 func docrawl(url string) []*Page {
 	// our result structure
 	var sitemap []*Page
 
 	// what we've already parsed, what we still need to parse
-	parsed := make(map[string]*Page)
+	crawled := make(map[string]*Page)
-	var queued []*Page
+	queued := make(map[string]*Page)
 
 	// create first page and add to the queue
-	u, _ := resolveURL("", url) // TODO check error
+	homepage, _ := NewPage(nil, url) // TODO check error
-	queued = append(queued, &Page{URL: u})
+	queued[homepage.URL.String()] = homepage
 
 	// main loop
 	for len(queued) > 0 {
 		// pop a page off of queued
 		var dest *Page
-		dest, queued = queued[len(queued)-1], queued[:len(queued)-1]
+		var destURL string
+		for destURL, dest = range queued {
+			break
+		}
+		delete(queued, destURL)
 
 		// skip loop if we already did this URL
-		if _, ok := parsed[dest.URL.String()]; ok {
+		if _, ok := crawled[dest.URL.String()]; ok {
 			continue // TODO handle
 		}
 
 		// mark this page as complete
-		parsed[dest.URL.String()] = dest
+		crawled[dest.URL.String()] = dest
 
 		// fetch page
 		text, err := fetchPage(dest)
@@ -64,14 +40,21 @@ func docrawl(url string) []*Page {
 			continue // TODO handle
 		}
 
-		// parse it
+		// parse links
 		_, links, err := parseLinks(text)
 		if err != nil {
 			continue // TODO handle
 		}
 		for _, l := range links {
-			u, _ := resolveURL(dest.URL.String(), l) // TODO check error
+			p, _ := NewPage(dest, l) // TODO check error
-			dest.Children = append(dest.Children, &Page{URL: u, Base: dest})
+			dest.Children = append(dest.Children, p)
+
+			// see if we should queue this new page for crawling
+			_, haveCrawled := crawled[p.URL.String()]
+			_, haveQueued := queued[p.URL.String()]
+			if !haveCrawled && !haveQueued {
+				queued[p.URL.String()] = p
+			}
 		}
 
 		// add page to list

src/docrawler/docrawler_test.go

@@ -78,14 +78,14 @@ func TestSimpleFetch(t *testing.T) {
 func TestSimpleMap(t *testing.T) {
 	pages := docrawl(baseURL)
 	t.Log("pages = ", pages)
-	if len(pages) != 1 {
+	if len(pages) != 4 {
 		t.Logf("got %v, wanted %v\n", len(pages), 1)
 		t.Fatal("got wrong number of pages")
 	}
 	if pages[0].URL.String() != baseURL {
 		t.Error("page URL is invalid")
 	}
-	if len(pages[0].Children) != 2 {
+	if len(pages[0].Children) != 3 {
 		t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1)
 		t.Fatal("got wrong number of links")
 	}
@@ -99,6 +99,11 @@ func TestSimpleMap(t *testing.T) {
 		t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
 		t.Error("link name is incorrect")
 	}
+	if pages[0].Children[2].URL.String() != baseURL+"scripts/blah.js" {
+		t.Logf("   Got: %q\n", pages[0].Children[2].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+		t.Error("link name is incorrect")
+	}
 }
 
 // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results

src/docrawler/page.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"net/url"
+)
+
+// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+type Page struct {
+	URL       *url.URL
+	Base      *Page
+	Title     string
+	MediaType string
+	Children  []*Page
+}
+
+// NewPage takes a referring Page + a URL and returns a new &Page{}
+func NewPage(referrer *Page, rawurl string) (*Page, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.URL.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// create struct and return
+	return &Page{URL: u}, nil
+}

src/docrawler/testsite/index.html

@@ -3,6 +3,7 @@
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script src="scripts/blah.js"/>
 	</body>
 </html>

src/docrawler/testsite/scripts/blah.js

@@ -0,0 +1 @@
+