From 249aca229b6196e2cafce24fcb29c5f55f2dc364 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 00:43:50 -0800 Subject: [PATCH 1/8] flow: Created branch 'feature/create_simple_sitemap'. From 5ef46be8c4d3d44181971dced04d90d4a5832c8f Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 00:58:01 -0800 Subject: [PATCH 2/8] add (presently failing) tests for discovering a simple site map --- src/docrawler/docrawler.go | 13 +++++++++++++ src/docrawler/docrawler_test.go | 31 +++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index c8f4ae0..1134f4a 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,5 +2,18 @@ package main import () +// Page is a struct which defines a single page, which URLs (links and assets) it contains +type Page struct { + URL string + Assets []string + Links []string +} + +// docrawl begins crawling the site at "url" +func docrawl(url string) []Page { + var result []Page + return result +} + func main() { } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 10dae64..4b491fb 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -44,7 +44,7 @@ func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error(err) + t.Fatal(err) } } @@ -53,7 +53,7 @@ func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) - t.Error("request for non existent file didn't fail properly") + t.Fatal("request for non existent file didn't fail properly") } } @@ -61,7 +61,7 @@ func TestBadRequest(t *testing.T) { func TestSimpleFetch(t *testing.T) { body, err := fetch(baseURL + "fetch_test.html") if err != nil { - t.Error(err) + t.Fatal(err) } desired := "\n" if body != desired { @@ -86,7 +86,7 @@ func TestSimpleParse(t *testing.T) { t.Error(err) } if len(matches) != 2 { - t.Error("invalid number of matches in parse") + t.Fatal("invalid number of matches in parse") } if matches[0] != "/about.html" { t.Error("match text is invalid") @@ -95,3 +95,26 @@ func TestSimpleParse(t *testing.T) { t.Error("match text is invalid") } } + +// TestSimpleMap figures out the site map for the site in baseURL +func TestSimpleMap(t *testing.T) { + pages := docrawl(baseURL) + if len(pages) != 1 { + t.Fatal("got wrong number of pages") + } + if pages[0].URL != baseURL { + t.Error("page URL is invalid") + } + if len(pages[0].Assets) != 1 { + t.Fatal("got wrong number of assets") + } + if pages[0].Assets[0] != baseURL+"/assets/image.png" { + t.Error("asset name is incorrect") + } + if len(pages[0].Links) != 1 { + t.Fatal("got wrong number of links") + } + if pages[0].Links[0] != baseURL+"/about.html" { + t.Error("link name is incorrect") + } +} From 0456723f0ba510030c2e2f6ceee6a9d5bfacbe68 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 12:12:30 -0800 Subject: [PATCH 3/8] minimally functional crawler - crawls single page, finds links, stores in data structure * refined Page struct. now need to refactor everything else to use this * wrote docrawl() function. NOT intended to be anything like the final design. just sketching it out to get a better idea of needs, design, etc. * replaced fmt.Print* with t.Log* in tests (oops!) * added a header fetcher function to get the file type (since we only want to parse html (with tests) * added small test for the URL resolver where the base URL is empty * added placeholder files in testsite/ for about.html and image.png --- src/docrawler/docrawler.go | 64 ++++++++++++++++++++++++++---- src/docrawler/docrawler_test.go | 45 ++++++++++++++------- src/docrawler/fetch.go | 31 +++++++++++++++ src/docrawler/testsite/about.html | 0 src/docrawler/testsite/image.png | Bin 0 -> 938 bytes src/docrawler/url_test.go | 10 ++--- 6 files changed, 123 insertions(+), 27 deletions(-) create mode 100644 src/docrawler/testsite/about.html create mode 100644 src/docrawler/testsite/image.png diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 1134f4a..508fe56 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,18 +1,66 @@ package main -import () +import ( + "net/url" +) -// Page is a struct which defines a single page, which URLs (links and assets) it contains +// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { - URL string - Assets []string - Links []string + URL *url.URL + Base *Page + Title string + MediaType string + Children []*Page } // docrawl begins crawling the site at "url" -func docrawl(url string) []Page { - var result []Page - return result +func docrawl(url string) []*Page { + // our result structure + var sitemap []*Page + + // what we've already parsed, what we still need to parse + parsed := make(map[string]*Page) + var queued []*Page + + // create first page and add to the queue + u, _ := resolveURL("", url) // TODO check error + queued = append(queued, &Page{URL: u}) + + // main loop + for len(queued) > 0 { + // pop a page off of queued + var dest *Page + dest, queued = queued[len(queued)-1], queued[:len(queued)-1] + + // skip loop if we already did this URL + if _, ok := parsed[dest.URL.String()]; ok { + continue // TODO handle + } + + // mark this page as complete + parsed[dest.URL.String()] = dest + + // fetch page + text, err := fetch(dest.URL.String()) + if err != nil { + continue // TODO handle + } + + // parse it + _, links, err := parse(text) + if err != nil { + continue // TODO handle + } + for _, l := range links { + u, _ := resolveURL(dest.URL.String(), l) // TODO check error + dest.Children = append(dest.Children, &Page{URL: u, Base: dest}) + } + + // add page to list + sitemap = append(sitemap, dest) + } + + return sitemap } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index eacf080..e3ce469 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net" "net/http" "os" @@ -43,7 +42,7 @@ func TestMain(m *testing.M) { func TestServerRunning(t *testing.T) { resp, err := http.Get(baseURL) if resp.StatusCode != 200 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal(err) } } @@ -52,7 +51,7 @@ func TestServerRunning(t *testing.T) { func TestBadRequest(t *testing.T) { resp, err := http.Get(baseURL + "doesnotexist.html") if resp.StatusCode != 404 || err != nil { - fmt.Printf("got %v, err = %v\n", resp.StatusCode, err) + t.Logf("got %v, err = %v\n", resp.StatusCode, err) t.Fatal("request for non existent file didn't fail properly") } } @@ -65,8 +64,8 @@ func TestSimpleFetch(t *testing.T) { } desired := "\n" if body != desired { - fmt.Printf(" Got: '%v'\n", body) - fmt.Printf("Wanted: '%v'\n", desired) + t.Logf(" Got: '%v'\n", body) + t.Logf("Wanted: '%v'\n", desired) t.Error("fetch succeeded, but content mismatched") } } @@ -74,22 +73,40 @@ func TestSimpleFetch(t *testing.T) { // TestSimpleMap figures out the site map for the site in baseURL func TestSimpleMap(t *testing.T) { pages := docrawl(baseURL) + t.Log("pages = ", pages) if len(pages) != 1 { + t.Logf("got %v, wanted %v\n", len(pages), 1) t.Fatal("got wrong number of pages") } - if pages[0].URL != baseURL { + if pages[0].URL.String() != baseURL { t.Error("page URL is invalid") } - if len(pages[0].Assets) != 1 { - t.Fatal("got wrong number of assets") - } - if pages[0].Assets[0] != baseURL+"/assets/image.png" { - t.Error("asset name is incorrect") - } - if len(pages[0].Links) != 1 { + if len(pages[0].Children) != 2 { + t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1) t.Fatal("got wrong number of links") } - if pages[0].Links[0] != baseURL+"/about.html" { + if pages[0].Children[0].URL.String() != baseURL+"about.html" { + t.Logf(" Got: %q\n", pages[0].Children[0].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"about.html") + t.Error("link name is incorrect") + } + if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" { + t.Logf(" Got: %q\n", pages[0].Children[1].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"assets/image.png") t.Error("link name is incorrect") } } + +// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +func TestHeaderFetching(t *testing.T) { + if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + t.Error("problem fetching filetype") + } + if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { + t.Logf("got %q, wanted %q", ft, "text/plain") + t.Error("problem fetching filetype") + } +} diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 4142836..72c5c24 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -1,12 +1,43 @@ package main import ( + "errors" "io/ioutil" + "mime" "net/http" ) +var ( + ErrContentTypeNotFound = errors.New("no Content-Type header found") +) + +func fetch_filetype(url string) (string, error) { + resp, err := http.Head(url) + if err != nil { + return "", err + } + + // pull the content type out of the http header + contentType, ok := resp.Header["Content-Type"] + if !ok { + // no Content-Type header found + return "", ErrContentTypeNotFound + } + + // parse mime type + mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return "", err + } + + return mediatype, nil +} + // fetch takes a URL, GETs it, and returns the body as a string func fetch(url string) (string, error) { + // figure out the file type + //filetype, err := fetch_filetype(url) + // GET the url resp, err := http.Get(url) if err != nil { diff --git a/src/docrawler/testsite/about.html b/src/docrawler/testsite/about.html new file mode 100644 index 0000000..e69de29 diff --git a/src/docrawler/testsite/image.png b/src/docrawler/testsite/image.png new file mode 100644 index 0000000000000000000000000000000000000000..db549f48d5adf6e823769c202a808700c31b5729 GIT binary patch literal 938 zcmaJ=O^ee&7!KQu6!stpew>EvWd)l_(oNfhW=op1Zorh%4cen=GTjc{Wa4CM)1DR& zg5clStAD_Q9`xW}@ajbb@ec^=q}z5qSaZm{@5}Q%@5j9Jrr&$KUw%*)1YzGjaR&S> z?DDNW{@?2lfAHfz>kQc`8MApnkzhq+gn%0dXJ~+eXz}6;Y6?Q>5c@+m^qv?YiKU>x zNO_!cwjeZHc^ZUM#J~uh;lvbwz5gHr9GT)X&6B;Z%>!OBUAJ~Kk#C@V*xu1ktmtLybgK;=G^ z8&H)ivTkTqLsr516L~fojg5iRUFYJSDNYzm4G8D+xiqgzgq}e~*L5hXP*p1&QOOnw z3-U^m9c&aFl!X+h43h*DiouA?m?`o~uUm-It!$F5$HWH)^B{$aBo{4hfS&h%XdG|R z85^LRdjBb$`HK|60m{gXhWz5j2SuvXuqg@{p*|t!n<@4ugpq7QQefMwsewZ;2ys#b zAFf!QXShkmf+R$)V~Siy!ZUALT?t|*-&t6OVZ)rO-g9i=O`wGG!H;Vec8 z+i;^BuCJ9eM!Rci?BqgT3@onquj}4~_zo53 zc6qt8U4NADou_=cKYyn``NO#9I+mY*9sRz#YL$|YpDymc_3?#J5-z`$Zoj_s^s;z8 MZl~wGYd?SW2WXWc6aWAK literal 0 HcmV?d00001 diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index 5381b64..38379df 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/url" "testing" ) @@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) { } cleanURL(u) if u.Host != "somehost.com:8765" { - fmt.Printf(" Got: %q\n", u.Host) - fmt.Printf("Wanted: %q\n", "somehost.com:8765") + t.Logf(" Got: %q\n", u.Host) + t.Logf("Wanted: %q\n", "somehost.com:8765") t.Error("host extraction failed") } if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" { @@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) { t.Error("resolveURL failed (probably parsing)") } if u.String() != wanted { - fmt.Printf(" Got: %q\n", u) - fmt.Printf("Wanted: %q\n", wanted) + t.Logf(" Got: %q\n", u) + t.Logf("Wanted: %q\n", wanted) t.Error("url resolution with base URL failed") } } @@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) { doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png") doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png") doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html") + doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html") } From 614f05f1d701c70446aa92e98aa8fbef3695c363 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 14:02:08 -0800 Subject: [PATCH 4/8] minor Makefile cleanup --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87413cf..8517d4f 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,15 @@ # derived from: http://zduck.com/2014/go-project-structure-and-dependencies/ # build targets (which aren't files) -.PHONY: build fmt lint vet test autotest cover run clean env vendor +.PHONY: build fmt lint vet test autotest cover doc run clean env vendor # configuration APPNAME := docrawler VENDOR := .vendor PACKAGES := \ - golang.org/x/tools/cmd/vet \ golang.org/x/tools/cmd/cover \ + golang.org/x/tools/cmd/godoc \ + golang.org/x/tools/cmd/vet \ github.com/golang/lint/golint # override GOPATH to be our vendor directory @@ -26,7 +27,7 @@ fmt: go fmt ./src/... lint: - # TODO golint is not working from inside the make environment for some reason + # TODO golint is not accepting the "./src/..." for some reason (from within make) golint ./src/${APPNAME}/*.go vet: @@ -42,6 +43,9 @@ autotest: cover: go test -coverprofile cover.out ./src/... && go tool cover -html cover.out +doc: + godoc -http=:6060 -index + run: build ./bin/${APPNAME} @@ -56,7 +60,7 @@ env: # * fetches all packages in ${PACKAGES} (GOPATH hardcoded to vendor directory only) # * removes source control dirs so packages are vendorable (commitable) vendor: - rm -dRf ./${VENDOR}/src && \ + rm -dRf ./${VENDOR} && \ GOPATH=${PWD}/${VENDOR} go get -u ${PACKAGES} && \ rm -rf `find ./${VENDOR}/src -type d -name .git` && \ rm -rf `find ./${VENDOR}/src -type d -name .hg` && \ From 0c48b94ece50604c49dc7fa240a33b269f58d5bd Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 14:54:52 -0800 Subject: [PATCH 5/8] refactoring and cleanup. needed before proceeding. * fetch() renamed to fetchPage. now takes *Page instead of url string * fetch_filetype() renamed to fetchFiletype. now takes *Page instead of url string * parse() renamed to parseLinks() * tests updated appropriately * minor cleanup (lint) * NewPage added for conveniently creating new *Page structs from url strings --- src/docrawler/docrawler.go | 22 +++++++++++++++++++-- src/docrawler/docrawler_test.go | 30 ++++++++++++++++++++++------ src/docrawler/fetch.go | 35 ++++++++++++++++++++------------- src/docrawler/parse.go | 4 ++-- src/docrawler/parse_test.go | 10 +++++++--- src/docrawler/url.go | 4 ++-- 6 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 508fe56..1b4b63f 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -13,6 +13,24 @@ type Page struct { Children []*Page } +// NewPage takes a referring Page + a URL and returns a new &Page{} +func NewPage(referrer *Page, rawurl string) (*Page, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.URL.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // create struct and return + return &Page{URL: u}, nil +} + // docrawl begins crawling the site at "url" func docrawl(url string) []*Page { // our result structure @@ -41,13 +59,13 @@ func docrawl(url string) []*Page { parsed[dest.URL.String()] = dest // fetch page - text, err := fetch(dest.URL.String()) + text, err := fetchPage(dest) if err != nil { continue // TODO handle } // parse it - _, links, err := parse(text) + _, links, err := parseLinks(text) if err != nil { continue // TODO handle } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index e3ce469..1641c06 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,7 +58,11 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - body, err := fetch(baseURL + "fetch_test.html") + page, err := NewPage(nil, baseURL+"fetch_test.html") + if err != nil { + t.Error("problem creating New Page struct") + } + body, err := fetchPage(page) if err != nil { t.Fatal(err) } @@ -97,16 +101,30 @@ func TestSimpleMap(t *testing.T) { } } -// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + basepage, err := NewPage(nil, baseURL) + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + + page, err := NewPage(basepage, "about.html") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { - t.Logf("got %q, wanted %q", ft, "text/plain") + + page, err = NewPage(basepage, "assets/image.png") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" { + t.Logf("got %q, wanted %q", page.MediaType, "text/plain") t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 72c5c24..7a4cbdf 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -8,38 +8,45 @@ import ( ) var ( - ErrContentTypeNotFound = errors.New("no Content-Type header found") + errContentTypeNotFound = errors.New("no Content-Type header found") ) -func fetch_filetype(url string) (string, error) { - resp, err := http.Head(url) +// fetchFiletype performs an http HEAD to get the media type, and sets it +// directly in Page.MediaType +func fetchFiletype(page *Page) error { + resp, err := http.Head(page.URL.String()) if err != nil { - return "", err + return err } // pull the content type out of the http header contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found - return "", ErrContentTypeNotFound + return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return err + } + + // set and return + page.MediaType = mediatype + return nil +} + +// fetchPage takes a Page, GETs it, and returns the body as a string +func fetchPage(page *Page) (string, error) { + // figure out the file type + err := fetchFiletype(page) if err != nil { return "", err } - return mediatype, nil -} - -// fetch takes a URL, GETs it, and returns the body as a string -func fetch(url string) (string, error) { - // figure out the file type - //filetype, err := fetch_filetype(url) - // GET the url - resp, err := http.Get(url) + resp, err := http.Get(page.URL.String()) if err != nil { return "", err } diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 13a1010..b951769 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -10,9 +10,9 @@ import ( var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) -// parse takes a string and attempts to parse any html title and all links out of it, +// parseLinks takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found -func parse(s string) (string, []string, error) { +func parseLinks(s string) (string, []string, error) { // find the title, or default to empty title := "" titleMatches := reTitle.FindStringSubmatch(s) diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index 9089d55..64df379 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -12,17 +12,18 @@ func TestSimpleParse(t *testing.T) { - + fooey! +