refactoring and cleanup. needed before proceeding.

* fetch() renamed to fetchPage. now takes *Page instead of url string
* fetch_filetype() renamed to fetchFiletype. now takes *Page instead of url string
* parse() renamed to parseLinks()
* tests updated appropriately
* minor cleanup (lint)
* NewPage added for conveniently creating new *Page structs from url strings

src/docrawler/docrawler.go

@@ -13,6 +13,24 @@ type Page struct {
 	Children  []*Page
 }
 
+// NewPage takes a referring Page + a URL and returns a new &Page{}
+func NewPage(referrer *Page, rawurl string) (*Page, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.URL.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// create struct and return
+	return &Page{URL: u}, nil
+}
+
 // docrawl begins crawling the site at "url"
 func docrawl(url string) []*Page {
 	// our result structure
@@ -41,13 +59,13 @@ func docrawl(url string) []*Page {
 		parsed[dest.URL.String()] = dest
 
 		// fetch page
-		text, err := fetch(dest.URL.String())
+		text, err := fetchPage(dest)
 		if err != nil {
 			continue // TODO handle
 		}
 
 		// parse it
-		_, links, err := parse(text)
+		_, links, err := parseLinks(text)
 		if err != nil {
 			continue // TODO handle
 		}

src/docrawler/docrawler_test.go

@@ -58,7 +58,11 @@ func TestBadRequest(t *testing.T) {
 
 // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
 func TestSimpleFetch(t *testing.T) {
-	body, err := fetch(baseURL + "fetch_test.html")
+	page, err := NewPage(nil, baseURL+"fetch_test.html")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	body, err := fetchPage(page)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -97,16 +101,30 @@ func TestSimpleMap(t *testing.T) {
 	}
 }
 
-// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results
+// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
 func TestHeaderFetching(t *testing.T) {
-	if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" {
+	basepage, err := NewPage(nil, baseURL)
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" {
 		t.Error("problem fetching filetype")
 	}
-	if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" {
+
+	page, err := NewPage(basepage, "about.html")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" {
 		t.Error("problem fetching filetype")
 	}
-	if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" {
-		t.Logf("got %q, wanted %q", ft, "text/plain")
+
+	page, err = NewPage(basepage, "assets/image.png")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" {
+		t.Logf("got %q, wanted %q", page.MediaType, "text/plain")
 		t.Error("problem fetching filetype")
 	}
 }

src/docrawler/fetch.go

@@ -8,38 +8,45 @@ import (
 )
 
 var (
-	ErrContentTypeNotFound = errors.New("no Content-Type header found")
+	errContentTypeNotFound = errors.New("no Content-Type header found")
 )
 
-func fetch_filetype(url string) (string, error) {
-	resp, err := http.Head(url)
+// fetchFiletype performs an http HEAD to get the media type, and sets it
+// directly in Page.MediaType
+func fetchFiletype(page *Page) error {
+	resp, err := http.Head(page.URL.String())
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// pull the content type out of the http header
 	contentType, ok := resp.Header["Content-Type"]
 	if !ok {
 		// no Content-Type header found
-		return "", ErrContentTypeNotFound
+		return errContentTypeNotFound
 	}
 
 	// parse mime type
 	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		return err
+	}
+
+	// set and return
+	page.MediaType = mediatype
+	return nil
+}
+
+// fetchPage takes a Page, GETs it, and returns the body as a string
+func fetchPage(page *Page) (string, error) {
+	// figure out the file type
+	err := fetchFiletype(page)
 	if err != nil {
 		return "", err
 	}
 
-	return mediatype, nil
-}
-
-// fetch takes a URL, GETs it, and returns the body as a string
-func fetch(url string) (string, error) {
-	// figure out the file type
-	//filetype, err := fetch_filetype(url)
-
 	// GET the url
-	resp, err := http.Get(url)
+	resp, err := http.Get(page.URL.String())
 	if err != nil {
 		return "", err
 	}

src/docrawler/parse.go

@@ -10,9 +10,9 @@ import (
 var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
 var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html title and all links out of it,
+// parseLinks takes a string and attempts to parse any html title and all links out of it,
 // and returns a sorted slice of the captures found
-func parse(s string) (string, []string, error) {
+func parseLinks(s string) (string, []string, error) {
 	// find the title, or default to empty
 	title := ""
 	titleMatches := reTitle.FindStringSubmatch(s)

src/docrawler/parse_test.go

@@ -12,17 +12,18 @@ func TestSimpleParse(t *testing.T) {
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	title, matches, err := parse(doc)
+	title, matches, err := parseLinks(doc)
 	if err != nil {
 		t.Error(err)
 	}
 	if title != "Test Page" {
 		t.Error("got wrong title")
 	}
-	if len(matches) != 2 {
+	if len(matches) != 3 {
 		t.Error("invalid number of matches in parse")
 	}
 	if matches[0] != "/about.html" {
@@ -31,4 +32,7 @@ func TestSimpleParse(t *testing.T) {
 	if matches[1] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
+	if matches[2] != "scripts/blah.js" {
+		t.Error("match text is invalid")
+	}
 }

src/docrawler/url.go

@@ -7,7 +7,7 @@ import (
 )
 
 var (
-	ErrInvalidURL = errors.New("this URL can't be parsed successfully")
+	errInvalidURL = errors.New("this URL can't be parsed successfully")
 )
 
 // cleanURL takes a URL and normalizes it by downcasing the host part
@@ -21,7 +21,7 @@ func checkURL(u *url.URL) error {
 	// check for non-empty Host and Scheme
 	if len(u.Host) == 0 || len(u.Scheme) == 0 {
 		// something about this URL doesn't meet our spec
-		return ErrInvalidURL
+		return errInvalidURL
 	}
 	return nil // success!
 }