From ccc65f9715e6887d51ce693061dce99c8508d080 Mon Sep 17 00:00:00 2001
From: Stephen Waits <steve@waits.net>
Date: Sun, 15 Feb 2015 16:03:10 -0800
Subject: [PATCH] coverage improved from 85.2% to 91.1%

---
 src/docrawler/docrawler.go        |  5 -----
 src/docrawler/fetch_test.go       | 27 +++++++++++++++++++++++++++
 src/docrawler/page_test.go        | 13 +++++++++++++
 src/docrawler/testsite/about.html | 11 +++++++++++
 src/docrawler/testsite/index.html |  1 +
 src/docrawler/url_test.go         | 16 ++++++++++++++++
 6 files changed, 68 insertions(+), 5 deletions(-)
 create mode 100644 src/docrawler/fetch_test.go
 create mode 100644 src/docrawler/page_test.go

diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go
index a8d9dcb..f66e9e0 100644
--- a/src/docrawler/docrawler.go
+++ b/src/docrawler/docrawler.go
@@ -25,11 +25,6 @@ func docrawl(url string) []*Page {
 		}
 		delete(queued, destURL)
 
-		// skip loop if we already did this URL
-		if _, ok := crawled[dest.URL.String()]; ok {
-			continue // TODO handle
-		}
-
 		// mark this page as complete
 		crawled[dest.URL.String()] = dest
 
diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go
new file mode 100644
index 0000000..7cdff84
--- /dev/null
+++ b/src/docrawler/fetch_test.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedHeaderFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if err := fetchFiletype(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchFiletype")
+	}
+}
+
+// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
+func TestFailedFetching(t *testing.T) {
+	page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
+	if err != nil {
+		t.Error("problem creating New Page struct")
+	}
+	if _, err := fetchPage(page); err == nil {
+		t.Error("tired fetching bogus page but didn't get nil back from fetchPage")
+	}
+}
diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go
new file mode 100644
index 0000000..217f290
--- /dev/null
+++ b/src/docrawler/page_test.go
@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestBadNewPage tests NewPage to see if we are getting expected failures
+func TestBadNewPage(t *testing.T) {
+	_, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
+	if err == nil {
+		t.Error("tried creating a NewPage with an invalid URL but didn't get an error")
+	}
+}
diff --git a/src/docrawler/testsite/about.html b/src/docrawler/testsite/about.html
index e69de29..fec8d53 100644
--- a/src/docrawler/testsite/about.html
+++ b/src/docrawler/testsite/about.html
@@ -0,0 +1,11 @@
+<html>
+	<head>
+		<title>About Test</title>
+		<script src="scripts/blah.js"/>
+	</head>
+	<body>
+		<img src="/assets/image.png"/>
+		<a href="/">fooey!</a>
+		<a href="http://doesntexist23492387492837492374982734.com/">bad one</a>
+	</body>
+</html>
diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html
index d75978e..ac60766 100644
--- a/src/docrawler/testsite/index.html
+++ b/src/docrawler/testsite/index.html
@@ -1,5 +1,6 @@
 <html>
 	<head>
+		<title>Home</title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go
index 38379df..b0867e8 100644
--- a/src/docrawler/url_test.go
+++ b/src/docrawler/url_test.go
@@ -52,3 +52,19 @@ func TestURLResolution(t *testing.T) {
 	doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
 	doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
 }
+
+// TestBadResolve tries to resolve bogus URLs to be sure we're getting errors
+/* TODO not sure how to make url.Parse fail in these cases!!!
+func TestBadResolve(t* testing.T) {
+	u, err := resolveURL("http://a.com/", "h%20t\tp://doesnte%20xist23492387492837492374982734.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+	u, err = resolveURL("h%20t\tp://doesnte%20xist23492387492837492374982734.com/", "http://a.com/")
+	if err == nil {
+		t.Logf("got %q", u.String())
+		t.Error("tried resolveURL with an invalid URL but didn't get an error")
+	}
+}
+*/