renamed Page -> httpItem, and changed its members to unexported as well

* tests all passing

src/docrawler/docrawler.go

@@ -5,23 +5,27 @@ import (
 	"os"
 )
 
+type itemSlice []*httpItem
+type itemMap   map[string]*httpItem
+
+
 // docrawl begins crawling the site at "url"
-func docrawl(url string) []*Page {
+func docrawl(url string) itemSlice {
 	// our result structure
-	var sitemap []*Page
+	var sitemap itemSlice
 
 	// what we've already parsed, what we still need to parse
-	crawled := make(map[string]*Page)
+	crawled := make(itemMap)
-	queued := make(map[string]*Page)
+	queued := make(itemMap)
 
 	// create first page and add to the queue
-	homepage, _ := NewPage(nil, url) // TODO check error
+	homepage, _ := newHTTPItem(nil, url) // TODO check error
-	queued[homepage.URL.String()] = homepage
+	queued[homepage.url.String()] = homepage
 
 	// main loop
 	for len(queued) > 0 {
 		// pop a page off of queued
-		var dest *Page
+		var dest *httpItem
 		var destURL string
 		for destURL, dest = range queued {
 			break
@@ -29,52 +33,52 @@ func docrawl(url string) []*Page {
 		delete(queued, destURL)
 
 		// mark this page as complete
-		crawled[dest.URL.String()] = dest
+		crawled[dest.url.String()] = dest
 
 		// add page to list
 		sitemap = append(sitemap, dest)
 
 		// make sure this URL has the same hostname as our first page
-		if dest.URL.Host != homepage.URL.Host {
+		if dest.url.Host != homepage.url.Host {
 			// skip URLs associated with other Hosts
-			dest.Skipped = true
+			dest.skipped = true
 			continue
 		}
 
 		// fetch page
 		text, err := fetchPage(dest)
 		if err != nil {
-			dest.Broken = true
+			dest.broken = true
 			continue
 		}
 
 		// parse links
 		title, links, err := parseLinks(text)
 		if err != nil {
-			dest.Broken = true
+			dest.broken = true
 			continue
 		}
-		dest.Title = title
+		dest.title = title
 		for _, l := range links {
-			p, err := NewPage(dest, l)
+			p, err := newHTTPItem(dest, l)
 			if err != nil {
 				continue // TODO check error
 			}
 
 			// see if we already know about this page
-			pCrawled, haveCrawled := crawled[p.URL.String()]
+			pCrawled, haveCrawled := crawled[p.url.String()]
-			pQueued, haveQueued := queued[p.URL.String()]
+			pQueued, haveQueued := queued[p.url.String()]
 			if haveCrawled {
 				// add the previously crawled page to children
-				dest.Children = append(dest.Children, pCrawled)
+				dest.children = append(dest.children, pCrawled)
 			} else if haveQueued {
 				// add the previously queued page to children
-				dest.Children = append(dest.Children, pQueued)
+				dest.children = append(dest.children, pQueued)
 			} else {
 				// add this new page (never seen) to children
-				dest.Children = append(dest.Children, p)
+				dest.children = append(dest.children, p)
 				// .. and queue it up for crawling
-				queued[p.URL.String()] = p
+				queued[p.url.String()] = p
 			}
 		}
 	}

src/docrawler/docrawler_test.go

@@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) {
 
 // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
 func TestSimpleFetch(t *testing.T) {
-	page, err := NewPage(nil, baseURL+"fetch_test.html")
+	page, err := newHTTPItem(nil, baseURL+"fetch_test.html")
 	if err != nil {
-		t.Error("problem creating New Page struct")
+		t.Error("problem creating New httpItem struct")
 	}
 	body, err := fetchPage(page)
 	if err != nil {
@@ -82,19 +82,19 @@ func TestSimpleMap(t *testing.T) {
 	// cycle through every page, counting stuff, finding specific pages
 	// so that we can then verify certain aspects of each page
 	skipped, children := 0, 0
-	var homepage, aboutpage, skippage *Page
+	var homepage, aboutpage, skippage *httpItem
 	for _, p := range pages {
-		if p.Title == "Home" {
+		if p.title == "Home" {
 			homepage = p
-		} else if p.Title == "About Test" {
+		} else if p.title == "About Test" {
 			aboutpage = p
-		} else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" {
+		} else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" {
 			skippage = p
 		}
-		if p.Skipped {
+		if p.skipped {
 			skipped++
 		}
-		children += len(p.Children)
+		children += len(p.children)
 	}
 
 	// check page counts
@@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) {
 	}
 
 	// check first page URL
-	if homepage.URL.String() != baseURL {
+	if homepage.url.String() != baseURL {
 		t.Error("page URL is invalid")
 	}
 
 	// specific page number of children
-	if len(homepage.Children) != 4 {
+	if len(homepage.children) != 4 {
-		t.Logf("got %v, wanted %v\n", len(homepage.Children), 4)
+		t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
 		t.Fatal("got wrong number of links")
 	}
-	if len(aboutpage.Children) != 4 {
+	if len(aboutpage.children) != 4 {
-		t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4)
+		t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4)
 		t.Fatal("got wrong number of links")
 	}
 
 	// make sure remote pages were skipped
-	if homepage.Skipped != false {
+	if homepage.skipped != false {
-		t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String())
+		t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String())
 		t.Error("child was skipped")
 	}
-	if skippage.Skipped != true {
+	if skippage.skipped != true {
-		t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String())
+		t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String())
 		t.Error("child wasn't skipped")
 	}
 
 	// verify page content
 	// TODO find a better way to do this
-	//if homepage.Children[0].URL.String() != baseURL+"about.html" {
+	//if homepage.children[0].url.String() != baseURL+"about.html" {
-	//t.Logf("   Got: %q\n", homepage.Children[0].URL.String())
+	//t.Logf("   Got: %q\n", homepage.children[0].url.String())
 	//t.Logf("Wanted: %q\n", baseURL+"about.html")
 	//t.Error("link name is incorrect")
 	//}
-	//if homepage.Children[1].URL.String() != baseURL+"assets/image.png" {
+	//if homepage.children[1].url.String() != baseURL+"assets/image.png" {
-	//t.Logf("   Got: %q\n", homepage.Children[1].URL.String())
+	//t.Logf("   Got: %q\n", homepage.children[1].url.String())
 	//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
 	//t.Error("link name is incorrect")
 	//}
-	//if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" {
+	//if homepage.children[3].url.String() != baseURL+"scripts/blah.js" {
-	//t.Logf("   Got: %q\n", homepage.Children[3].URL.String())
+	//t.Logf("   Got: %q\n", homepage.children[3].url.String())
 	//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
 	//t.Error("link name is incorrect")
 	//}
@@ -157,28 +157,28 @@ func TestSimpleMap(t *testing.T) {
 
 // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
 func TestHeaderFetching(t *testing.T) {
-	basepage, err := NewPage(nil, baseURL)
+	basepage, err := newHTTPItem(nil, baseURL)
 	if err != nil {
-		t.Error("problem creating New Page struct")
+		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" {
+	if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" {
 		t.Error("problem fetching filetype")
 	}
 
-	page, err := NewPage(basepage, "about.html")
+	page, err := newHTTPItem(basepage, "about.html")
 	if err != nil {
-		t.Error("problem creating New Page struct")
+		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" {
+	if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" {
 		t.Error("problem fetching filetype")
 	}
 
-	page, err = NewPage(basepage, "assets/image.png")
+	page, err = newHTTPItem(basepage, "assets/image.png")
 	if err != nil {
-		t.Error("problem creating New Page struct")
+		t.Error("problem creating New httpItem struct")
 	}
-	if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" {
+	if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" {
-		t.Logf("got %q, wanted %q", page.MediaType, "image/png")
+		t.Logf("got %q, wanted %q", page.mediaType, "image/png")
 		t.Error("problem fetching filetype")
 	}
 }

src/docrawler/fetch.go

@@ -13,9 +13,9 @@ var (
 )
 
 // fetchFiletype performs an http HEAD to get the media type, and sets it
-// directly in Page.MediaType
+// directly in httpItem.mediaType
-func fetchFiletype(page *Page) error {
+func fetchFiletype(page *httpItem) error {
-	resp, err := http.Head(page.URL.String())
+	resp, err := http.Head(page.url.String())
 	if err != nil {
 		return err
 	}
@@ -39,12 +39,12 @@ func fetchFiletype(page *Page) error {
 	}
 
 	// set and return
-	page.MediaType = mediatype
+	page.mediaType = mediatype
 	return nil
 }
 
-// fetchPage takes a Page, GETs it, and returns the body as a string
+// fetchPage takes a httpItem, GETs it, and returns the body as a string
-func fetchPage(page *Page) (string, error) {
+func fetchPage(page *httpItem) (string, error) {
 	// figure out the file type
 	err := fetchFiletype(page)
 	if err != nil {
@@ -52,12 +52,12 @@ func fetchPage(page *Page) (string, error) {
 	}
 
 	// we only want to fetch html
-	if page.MediaType != "text/html" {
+	if page.mediaType != "text/html" {
 		return "", nil // but this isn't an error!
 	}
 
 	// GET the url
-	resp, err := http.Get(page.URL.String())
+	resp, err := http.Get(page.url.String())
 	if err != nil {
 		return "", err
 	}

src/docrawler/fetch_test.go

@@ -6,7 +6,7 @@ import (
 
 // TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
 func TestFailedHeaderFetching(t *testing.T) {
-	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}
@@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) {
 
 // TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
 func TestFailedFetching(t *testing.T) {
-	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
 	if err != nil {
 		t.Error("problem creating New Page struct")
 	}

src/docrawler/output.go

@@ -15,32 +15,32 @@ type Location struct {
 	Remote []string
 }
 
-func sitemapToLocations(pages []*Page) []*Location {
+func sitemapToLocations(pages []*httpItem) []*Location {
 	// build a map of our pages
-	pageMap := make(map[string]*Page)
+	pageMap := make(map[string]*httpItem)
 	for _, p := range pages {
-		pageMap[p.URL.String()] = p
+		pageMap[p.url.String()] = p
 	}
 
 	// build a slice of locations (one per page)
 	var locations []*Location
 	for _, p := range pages {
-		if p.MediaType == "text/html" {
+		if p.mediaType == "text/html" {
 			// create a location for this page
-			l := &Location{URL: p.URL.String(), Title: p.Title}
+			l := &Location{URL: p.url.String(), Title: p.title}
 
 			// add its children
-			for _, c := range p.Children {
+			for _, c := range p.children {
 				// look up this child's media type from the root list of pages
-				//mediaType := pageMap[c.URL.String()].MediaType
+				//mediaType := pageMap[c.url.String()].mediaType
-				if c.Skipped {
+				if c.skipped {
-					l.Remote = append(l.Remote, c.URL.String())
+					l.Remote = append(l.Remote, c.url.String())
-				} else if c.MediaType == "text/html" {
+				} else if c.mediaType == "text/html" {
-					l.Links = append(l.Links, c.URL.String())
+					l.Links = append(l.Links, c.url.String())
-				} else if c.Broken {
+				} else if c.broken {
-					l.Broken = append(l.Broken, c.URL.String())
+					l.Broken = append(l.Broken, c.url.String())
 				} else {
-					l.Assets = append(l.Assets, c.URL.String())
+					l.Assets = append(l.Assets, c.url.String())
 				}
 			}
 

src/docrawler/page.go

@@ -4,22 +4,22 @@ import (
 	"net/url"
 )
 
-// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc.
-type Page struct {
+type httpItem struct {
-	URL       *url.URL
+	url       *url.URL
-	Title     string
+	title     string
-	MediaType string
+	mediaType string
-	Skipped   bool
+	skipped   bool
-	Broken    bool
+	broken    bool
-	Children  []*Page
+	children  []*httpItem
 }
 
-// NewPage takes a referring Page + a URL and returns a new &Page{}
+// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{}
-func NewPage(referrer *Page, rawurl string) (*Page, error) {
+func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) {
 	// determine the base URL so we can resulve this into a full URL
 	baseURL := ""
 	if referrer != nil {
-		baseURL = referrer.URL.String()
+		baseURL = referrer.url.String()
 	}
 
 	// resolve URL into a full absolute URL (non-relative)
@@ -29,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) {
 	}
 
 	// create struct and return
-	return &Page{URL: u, Skipped: false, Broken: false}, nil
+	return &httpItem{url: u, skipped: false, broken: false}, nil
 }

src/docrawler/page_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 )
 
-// TestBadNewPage tests NewPage to see if we are getting expected failures
+// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures
-func TestBadNewPage(t *testing.T) {
+func TestBadnewHTTPItem(t *testing.T) {
-	_, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	_, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
 	if err == nil {
-		t.Error("tried creating a NewPage with an invalid URL but didn't get an error")
+		t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error")
 	}
 }