diff --git a/Makefile b/Makefile
index 6ab64a1..8517d4f 100644
--- a/Makefile
+++ b/Makefile
@@ -3,14 +3,15 @@
 # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
 
 # build targets (which aren't files)
-.PHONY: build fmt lint vet test autotest cover run clean env vendor
+.PHONY: build fmt lint vet test autotest cover doc run clean env vendor
 
 # configuration
 APPNAME := docrawler
 VENDOR := .vendor
 PACKAGES :=                     \
-	golang.org/x/tools/cmd/vet    \
 	golang.org/x/tools/cmd/cover  \
+	golang.org/x/tools/cmd/godoc  \
+	golang.org/x/tools/cmd/vet    \
 	github.com/golang/lint/golint
 
 # override GOPATH to be our vendor directory
@@ -26,7 +27,8 @@ fmt:
 	go fmt ./src/...
 
 lint:
-	golint ./src
+	# TODO golint is not accepting the "./src/..." for some reason (from within make)
+	golint ./src/${APPNAME}/*.go
 
 vet:
 	go vet ./src/...
@@ -41,6 +43,9 @@ autotest:
 cover:
 	go test -coverprofile cover.out ./src/...  && go tool cover -html cover.out
 
+doc:
+	godoc -http=:6060 -index
+
 run: build
 	./bin/${APPNAME}
 
@@ -55,7 +60,7 @@ env:
 # * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only)
 # * removes source control dirs so packages are vendorable (commitable)
 vendor:
-	rm -dRf ./${VENDOR}/src                          && \
+	rm -dRf ./${VENDOR}                              && \
 	GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES}    && \
 	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
 	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go
index c8f4ae0..a8d9dcb 100644
--- a/src/docrawler/docrawler.go
+++ b/src/docrawler/docrawler.go
@@ -2,5 +2,66 @@ package main
 
 import ()
 
+// docrawl begins crawling the site at "url"
+func docrawl(url string) []*Page {
+	// our result structure
+	var sitemap []*Page
+
+	// what we've already parsed, what we still need to parse
+	crawled := make(map[string]*Page)
+	queued := make(map[string]*Page)
+
+	// create first page and add to the queue
+	homepage, _ := NewPage(nil, url) // TODO check error
+	queued[homepage.URL.String()] = homepage
+
+	// main loop
+	for len(queued) > 0 {
+		// pop a page off of queued
+		var dest *Page
+		var destURL string
+		for destURL, dest = range queued {
+			break
+		}
+		delete(queued, destURL)
+
+		// skip loop if we already did this URL
+		if _, ok := crawled[dest.URL.String()]; ok {
+			continue // TODO handle
+		}
+
+		// mark this page as complete
+		crawled[dest.URL.String()] = dest
+
+		// fetch page
+		text, err := fetchPage(dest)
+		if err != nil {
+			continue // TODO handle
+		}
+
+		// parse links
+		_, links, err := parseLinks(text)
+		if err != nil {
+			continue // TODO handle
+		}
+		for _, l := range links {
+			p, _ := NewPage(dest, l) // TODO check error
+			dest.Children = append(dest.Children, p)
+
+			// see if we should queue this new page for crawling
+			_, haveCrawled := crawled[p.URL.String()]
+			_, haveQueued := queued[p.URL.String()]
+			if !haveCrawled && !haveQueued {
+				queued[p.URL.String()] = p
+			}
+		}
+
+		// add page to list
+		sitemap = append(sitemap, dest)
+	}
+
+	return sitemap
+}
+
 func main() {
 }
diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go
index 59d7aaa..e99935a 100644
--- a/src/docrawler/docrawler_test.go
+++ b/src/docrawler/docrawler_test.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -43,8 +42,8 @@ func TestMain(m *testing.M) {
 func TestServerRunning(t *testing.T) {
 	resp, err := http.Get(baseURL)
 	if resp.StatusCode != 200 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error(err)
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal(err)
 	}
 }
 
@@ -52,21 +51,85 @@ func TestServerRunning(t *testing.T) {
 func TestBadRequest(t *testing.T) {
 	resp, err := http.Get(baseURL + "doesnotexist.html")
 	if resp.StatusCode != 404 || err != nil {
-		fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
-		t.Error("request for non existent file didn't fail properly")
+		t.Logf("got %v, err = %v\n", resp.StatusCode, err)
+		t.Fatal("request for non existent file didn't fail properly")
 	}
 }
 
 // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
 func TestSimpleFetch(t *testing.T) {
-	body, err := fetch(baseURL + "fetch_test.html")
+	page, err := NewPage(nil, baseURL+"fetch_test.html")
 	if err != nil {
-		t.Error(err)
+		t.Error("problem creating New Page struct")
+	}
+	body, err := fetchPage(page)
+	if err != nil {
+		t.Fatal(err)
 	}
 	desired := "<html><head></head><body></body></html>\n"
 	if body != desired {
-		fmt.Printf("   Got: '%v'\n", body)
-		fmt.Printf("Wanted: '%v'\n", desired)
+		t.Logf("   Got: '%v'\n", body)
+		t.Logf("Wanted: '%v'\n", desired)
 		t.Error("fetch succeeded, but content mismatched")
 	}
 }
+
+// TestSimpleMap figures out the site map for the site in baseURL
+func TestSimpleMap(t *testing.T) {
+	pages := docrawl(baseURL)
+	t.Log("pages = ", pages)
+	if len(pages) != 4 {
+		t.Logf("got %v, wanted %v\n", len(pages), 1)
+		t.Fatal("got wrong number of pages")
+	}
+	if pages[0].URL.String() != baseURL {
+		t.Error("page URL is invalid")
+	}
+	if len(pages[0].Children) != 3 {
+		t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1)
+		t.Fatal("got wrong number of links")
+	}
+	if pages[0].Children[0].URL.String() != baseURL+"about.html" {
+		t.Logf("   Got: %q\n", pages[0].Children[0].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"about.html")
+		t.Error("link name is incorrect")
+	}
+	if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" {
+		t.Logf("   Got: %q\n", pages[0].Children[1].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
+		t.Error("link name is incorrect")
+	}
+	if pages[0].Children[2].URL.String() != baseURL+"scripts/blah.js" {
+		t.Logf("   Got: %q\n", pages[0].Children[2].URL.String())
+		t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
+		t.Error("link name is incorrect")
+	}
+}
+
+// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
+func TestHeaderFetching(t *testing.T) {
+	basepage, err := NewPage(nil, baseURL)
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err := NewPage(basepage, "about.html")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" {
+		t.Error("problem fetching filetype")
+	}
+
+	page, err = NewPage(basepage, "assets/image.png")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" {
+		t.Logf("got %q, wanted %q", page.MediaType, "text/plain")
+		t.Error("problem fetching filetype")
+	}
+}
diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go
index 4142836..7a4cbdf 100644
--- a/src/docrawler/fetch.go
+++ b/src/docrawler/fetch.go
@@ -1,14 +1,52 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
+	"mime"
 	"net/http"
 )
 
-// fetch takes a URL, GETs it, and returns the body as a string
-func fetch(url string) (string, error) {
+var (
+	errContentTypeNotFound = errors.New("no Content-Type header found")
+)
+
+// fetchFiletype performs an http HEAD to get the media type, and sets it
+// directly in Page.MediaType
+func fetchFiletype(page *Page) error {
+	resp, err := http.Head(page.URL.String())
+	if err != nil {
+		return err
+	}
+
+	// pull the content type out of the http header
+	contentType, ok := resp.Header["Content-Type"]
+	if !ok {
+		// no Content-Type header found
+		return errContentTypeNotFound
+	}
+
+	// parse mime type
+	mediatype, _, err := mime.ParseMediaType(contentType[0])
+	if err != nil {
+		return err
+	}
+
+	// set and return
+	page.MediaType = mediatype
+	return nil
+}
+
+// fetchPage takes a Page, GETs it, and returns the body as a string
+func fetchPage(page *Page) (string, error) {
+	// figure out the file type
+	err := fetchFiletype(page)
+	if err != nil {
+		return "", err
+	}
+
 	// GET the url
-	resp, err := http.Get(url)
+	resp, err := http.Get(page.URL.String())
 	if err != nil {
 		return "", err
 	}
diff --git a/src/docrawler/page.go b/src/docrawler/page.go
new file mode 100644
index 0000000..37a7f7c
--- /dev/null
+++ b/src/docrawler/page.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"net/url"
+)
+
+// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
+type Page struct {
+	URL       *url.URL
+	Base      *Page
+	Title     string
+	MediaType string
+	Children  []*Page
+}
+
+// NewPage takes a referring Page + a URL and returns a new &Page{}
+func NewPage(referrer *Page, rawurl string) (*Page, error) {
+	// determine the base URL so we can resulve this into a full URL
+	baseURL := ""
+	if referrer != nil {
+		baseURL = referrer.URL.String()
+	}
+
+	// resolve URL into a full absolute URL (non-relative)
+	u, err := resolveURL(baseURL, rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// create struct and return
+	return &Page{URL: u}, nil
+}
diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go
index 13a1010..b951769 100644
--- a/src/docrawler/parse.go
+++ b/src/docrawler/parse.go
@@ -10,9 +10,9 @@ import (
 var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
 var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`)
 
-// parse takes a string and attempts to parse any html title and all links out of it,
+// parseLinks takes a string and attempts to parse any html title and all links out of it,
 // and returns a sorted slice of the captures found
-func parse(s string) (string, []string, error) {
+func parseLinks(s string) (string, []string, error) {
 	// find the title, or default to empty
 	title := ""
 	titleMatches := reTitle.FindStringSubmatch(s)
diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go
index 9089d55..64df379 100644
--- a/src/docrawler/parse_test.go
+++ b/src/docrawler/parse_test.go
@@ -12,17 +12,18 @@ func TestSimpleParse(t *testing.T) {
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script href="scripts/blah.js"/>
 	</body>
 </html>`
-	title, matches, err := parse(doc)
+	title, matches, err := parseLinks(doc)
 	if err != nil {
 		t.Error(err)
 	}
 	if title != "Test Page" {
 		t.Error("got wrong title")
 	}
-	if len(matches) != 2 {
+	if len(matches) != 3 {
 		t.Error("invalid number of matches in parse")
 	}
 	if matches[0] != "/about.html" {
@@ -31,4 +32,7 @@ func TestSimpleParse(t *testing.T) {
 	if matches[1] != "/assets/image.png" {
 		t.Error("match text is invalid")
 	}
+	if matches[2] != "scripts/blah.js" {
+		t.Error("match text is invalid")
+	}
 }
diff --git a/src/docrawler/testsite/about.html b/src/docrawler/testsite/about.html
new file mode 100644
index 0000000..e69de29
diff --git a/src/docrawler/testsite/image.png b/src/docrawler/testsite/image.png
new file mode 100644
index 0000000..db549f4
Binary files /dev/null and b/src/docrawler/testsite/image.png differ
diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html
index 3a41cf9..d75978e 100644
--- a/src/docrawler/testsite/index.html
+++ b/src/docrawler/testsite/index.html
@@ -3,6 +3,7 @@
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
-		<a href="/about.html">
+		<a href="/about.html">fooey!</a>
+		<script src="scripts/blah.js"/>
 	</body>
 </html>
diff --git a/src/docrawler/testsite/scripts/blah.js b/src/docrawler/testsite/scripts/blah.js
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/src/docrawler/testsite/scripts/blah.js
@@ -0,0 +1 @@
+
diff --git a/src/docrawler/url.go b/src/docrawler/url.go
index 0e90021..d66bc44 100644
--- a/src/docrawler/url.go
+++ b/src/docrawler/url.go
@@ -7,7 +7,7 @@ import (
 )
 
 var (
-	ErrInvalidURL = errors.New("this URL can't be parsed successfully")
+	errInvalidURL = errors.New("this URL can't be parsed successfully")
 )
 
 // cleanURL takes a URL and normalizes it by downcasing the host part
@@ -21,7 +21,7 @@ func checkURL(u *url.URL) error {
 	// check for non-empty Host and Scheme
 	if len(u.Host) == 0 || len(u.Scheme) == 0 {
 		// something about this URL doesn't meet our spec
-		return ErrInvalidURL
+		return errInvalidURL
 	}
 	return nil // success!
 }
diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go
index 5381b64..38379df 100644
--- a/src/docrawler/url_test.go
+++ b/src/docrawler/url_test.go
@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net/url"
 	"testing"
 )
@@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) {
 	}
 	cleanURL(u)
 	if u.Host != "somehost.com:8765" {
-		fmt.Printf("   Got: %q\n", u.Host)
-		fmt.Printf("Wanted: %q\n", "somehost.com:8765")
+		t.Logf("   Got: %q\n", u.Host)
+		t.Logf("Wanted: %q\n", "somehost.com:8765")
 		t.Error("host extraction failed")
 	}
 	if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
@@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) {
 		t.Error("resolveURL failed (probably parsing)")
 	}
 	if u.String() != wanted {
-		fmt.Printf("   Got: %q\n", u)
-		fmt.Printf("Wanted: %q\n", wanted)
+		t.Logf("   Got: %q\n", u)
+		t.Logf("Wanted: %q\n", wanted)
 		t.Error("url resolution with base URL failed")
 	}
 }
@@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) {
 	doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
 	doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
 	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
+	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
 }