From 6665729e3f5ba4077daa2f9ea6044be8e54c2107 Mon Sep 17 00:00:00 2001
From: Stephen Waits <steve@waits.net>
Date: Mon, 16 Feb 2015 23:02:03 -0800
Subject: [PATCH] rename docrawl -> doCrawl; add some missing comments

---
 src/docrawler/docrawler.go      | 4 ++--
 src/docrawler/docrawler_test.go | 4 ++--
 src/docrawler/fetch.go          | 1 +
 src/docrawler/item.go           | 1 +
 src/docrawler/output.go         | 1 +
 src/docrawler/url.go            | 4 ++++
 src/docrawler/url_test.go       | 1 +
 src/docrawler/util_test.go      | 3 +++
 8 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go
index 3a72d18..2502944 100644
--- a/src/docrawler/docrawler.go
+++ b/src/docrawler/docrawler.go
@@ -8,7 +8,7 @@ import (
 )
 
 // docrawl begins crawling the site at "homeurl"
-func docrawl(homeurl string) itemSlice {
+func doCrawl(homeurl string) itemSlice {
 	// set of what we have already crawled
 	crawled := make(itemMap)
 
@@ -126,7 +126,7 @@ func crawlItem(item *httpItem, rchan chan<- *httpItem) {
 func main() {
 	// crawl each URL on the command line
 	for _, u := range os.Args[1:] {
-		pages := docrawl(u)
+		pages := doCrawl(u)
 		l := sitemapToLocations(pages)
 		j, _ := locationsToJSON(l)
 		fmt.Println(j)
diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go
index c908ba0..a5a22e9 100644
--- a/src/docrawler/docrawler_test.go
+++ b/src/docrawler/docrawler_test.go
@@ -76,7 +76,7 @@ func TestSimpleFetch(t *testing.T) {
 
 // TestSimpleMap figures out the site map for the site in baseURL
 func TestSimpleMap(t *testing.T) {
-	pages := docrawl(baseURL)
+	pages := doCrawl(baseURL)
 
 	// because our crawl is non-deterministic, we have to do a complete
 	// cycle through every page, counting stuff, finding specific pages
@@ -185,7 +185,7 @@ func TestHeaderFetching(t *testing.T) {
 
 // TestJsonOutput gets a sitemap and then converts it to json
 func TestJsonOutput(t *testing.T) {
-	pages := docrawl(baseURL)
+	pages := doCrawl(baseURL)
 	l := sitemapToLocations(pages)
 	if len(l) != 2 {
 		t.Error("sitemapToLocations has the wrong number of locations")
diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go
index 9cd67a6..bab34f0 100644
--- a/src/docrawler/fetch.go
+++ b/src/docrawler/fetch.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 )
 
+// custom errors
 var (
 	errContentTypeNotFound = errors.New("no Content-Type header found")
 	errFetchError          = errors.New("couldn't fetch item")
diff --git a/src/docrawler/item.go b/src/docrawler/item.go
index cb9d90c..eaf42e0 100644
--- a/src/docrawler/item.go
+++ b/src/docrawler/item.go
@@ -13,6 +13,7 @@ type itemMap map[string]*httpItem
 // itemType is an enum so we know how to classify each item
 type itemType int
 
+// enums for itemType
 const (
 	tUnknown itemType = iota
 	tHTMLPage
diff --git a/src/docrawler/output.go b/src/docrawler/output.go
index 58361bf..6691499 100644
--- a/src/docrawler/output.go
+++ b/src/docrawler/output.go
@@ -77,6 +77,7 @@ func sitemapToLocations(pages itemSlice) []*Location {
 	return locations
 }
 
+// locationsToJSON takes a *Location slice and marshals it into a JSON string
 func locationsToJSON(locations []*Location) (string, error) {
 	b, err := json.Marshal(locations)
 	if err != nil {
diff --git a/src/docrawler/url.go b/src/docrawler/url.go
index d98657a..54fe14a 100644
--- a/src/docrawler/url.go
+++ b/src/docrawler/url.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 )
 
+// custom errors
 var (
 	errInvalidURL = errors.New("this URL can't be parsed successfully")
 )
@@ -50,6 +51,9 @@ func resolveURL(referringURL string, currentURL string) (*url.URL, error) {
 	return uResolved, checkURL(uResolved)
 }
 
+// stripAnchorFromURL returns a version of the URL with out the "Fragment" part,
+// which is anything after the '#' character, so:
+// http://a.com/blah.html#anchor becomes http://a.com/blah.html
 func stripAnchorFromURL(u *url.URL) string {
 	// copy the url locally
 	ucopy := *u
diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go
index 07a4a9b..cd19076 100644
--- a/src/docrawler/url_test.go
+++ b/src/docrawler/url_test.go
@@ -69,6 +69,7 @@ func TestBadResolve(t* testing.T) {
 }
 */
 
+// TestURLStripAnchor tests the stripAnchorFromURL() function
 func TestURLStripAnchor(t *testing.T) {
 	u, err := url.Parse("http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo")
 	if err != nil {
diff --git a/src/docrawler/util_test.go b/src/docrawler/util_test.go
index 1ad3dca..fde2da2 100644
--- a/src/docrawler/util_test.go
+++ b/src/docrawler/util_test.go
@@ -5,6 +5,8 @@ import (
 	"testing"
 )
 
+// TestUniqStrings passes a slice of strings to uniqStrings() to see if it removes
+// all duplicates
 func TestUniqStrings(t *testing.T) {
 	strs := []string{"a", "a", "b", "a", "b", "c", "d", "a", "b", "e", "d"}
 	uniq := uniqStrings(strs)
@@ -17,6 +19,7 @@ func TestUniqStrings(t *testing.T) {
 	}
 }
 
+// TestUniqStringsNilParm passes a nil slice to uniqStrings to be sure it returns nil
 func TestUniqStringsNilParm(t *testing.T) {
 	uniq := uniqStrings(nil)
 	if uniq != nil {