From 0c48b94ece50604c49dc7fa240a33b269f58d5bd Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 14:54:52 -0800 Subject: [PATCH] refactoring and cleanup. needed before proceeding. * fetch() renamed to fetchPage. now takes *Page instead of url string * fetch_filetype() renamed to fetchFiletype. now takes *Page instead of url string * parse() renamed to parseLinks() * tests updated appropriately * minor cleanup (lint) * NewPage added for conveniently creating new *Page structs from url strings --- src/docrawler/docrawler.go | 22 +++++++++++++++++++-- src/docrawler/docrawler_test.go | 30 ++++++++++++++++++++++------ src/docrawler/fetch.go | 35 ++++++++++++++++++++------------- src/docrawler/parse.go | 4 ++-- src/docrawler/parse_test.go | 10 +++++++--- src/docrawler/url.go | 4 ++-- 6 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 508fe56..1b4b63f 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -13,6 +13,24 @@ type Page struct { Children []*Page } +// NewPage takes a referring Page + a URL and returns a new &Page{} +func NewPage(referrer *Page, rawurl string) (*Page, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.URL.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // create struct and return + return &Page{URL: u}, nil +} + // docrawl begins crawling the site at "url" func docrawl(url string) []*Page { // our result structure @@ -41,13 +59,13 @@ func docrawl(url string) []*Page { parsed[dest.URL.String()] = dest // fetch page - text, err := fetch(dest.URL.String()) + text, err := fetchPage(dest) if err != nil { continue // TODO handle } // parse it - _, links, err := parse(text) + _, links, err := parseLinks(text) if err != nil { continue // TODO handle } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index e3ce469..1641c06 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,7 +58,11 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - body, err := fetch(baseURL + "fetch_test.html") + page, err := NewPage(nil, baseURL+"fetch_test.html") + if err != nil { + t.Error("problem creating New Page struct") + } + body, err := fetchPage(page) if err != nil { t.Fatal(err) } @@ -97,16 +101,30 @@ func TestSimpleMap(t *testing.T) { } } -// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results +// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" { + basepage, err := NewPage(nil, baseURL) + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" { + + page, err := NewPage(basepage, "about.html") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { t.Error("problem fetching filetype") } - if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" { - t.Logf("got %q, wanted %q", ft, "text/plain") + + page, err = NewPage(basepage, "assets/image.png") + if err != nil { + t.Error("problem creating New Page struct") + } + if err := fetchFiletype(page); err != nil || page.MediaType != "text/plain" { + t.Logf("got %q, wanted %q", page.MediaType, "text/plain") t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 72c5c24..7a4cbdf 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -8,38 +8,45 @@ import ( ) var ( - ErrContentTypeNotFound = errors.New("no Content-Type header found") + errContentTypeNotFound = errors.New("no Content-Type header found") ) -func fetch_filetype(url string) (string, error) { - resp, err := http.Head(url) +// fetchFiletype performs an http HEAD to get the media type, and sets it +// directly in Page.MediaType +func fetchFiletype(page *Page) error { + resp, err := http.Head(page.URL.String()) if err != nil { - return "", err + return err } // pull the content type out of the http header contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found - return "", ErrContentTypeNotFound + return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) + if err != nil { + return err + } + + // set and return + page.MediaType = mediatype + return nil +} + +// fetchPage takes a Page, GETs it, and returns the body as a string +func fetchPage(page *Page) (string, error) { + // figure out the file type + err := fetchFiletype(page) if err != nil { return "", err } - return mediatype, nil -} - -// fetch takes a URL, GETs it, and returns the body as a string -func fetch(url string) (string, error) { - // figure out the file type - //filetype, err := fetch_filetype(url) - // GET the url - resp, err := http.Get(url) + resp, err := http.Get(page.URL.String()) if err != nil { return "", err } diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 13a1010..b951769 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -10,9 +10,9 @@ import ( var reURL = regexp.MustCompile(`(?i)(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) -// parse takes a string and attempts to parse any html title and all links out of it, +// parseLinks takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found -func parse(s string) (string, []string, error) { +func parseLinks(s string) (string, []string, error) { // find the title, or default to empty title := "" titleMatches := reTitle.FindStringSubmatch(s) diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index 9089d55..64df379 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -12,17 +12,18 @@ func TestSimpleParse(t *testing.T) { - + fooey! +