diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 1b4b63f..1c3cb8a 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,62 +1,38 @@ package main import ( - "net/url" ) -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Base *Page - Title string - MediaType string - Children []*Page -} - -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { - // determine the base URL so we can resulve this into a full URL - baseURL := "" - if referrer != nil { - baseURL = referrer.URL.String() - } - - // resolve URL into a full absolute URL (non-relative) - u, err := resolveURL(baseURL, rawurl) - if err != nil { - return nil, err - } - - // create struct and return - return &Page{URL: u}, nil -} - // docrawl begins crawling the site at "url" func docrawl(url string) []*Page { // our result structure var sitemap []*Page // what we've already parsed, what we still need to parse - parsed := make(map[string]*Page) - var queued []*Page + crawled := make(map[string]*Page) + queued := make(map[string]*Page) // create first page and add to the queue - u, _ := resolveURL("", url) // TODO check error - queued = append(queued, &Page{URL: u}) + homepage, _ := NewPage(nil, url) // TODO check error + queued[homepage.URL.String()] = homepage // main loop for len(queued) > 0 { // pop a page off of queued var dest *Page - dest, queued = queued[len(queued)-1], queued[:len(queued)-1] + var destURL string + for destURL, dest = range queued { + break + } + delete(queued, destURL) // skip loop if we already did this URL - if _, ok := parsed[dest.URL.String()]; ok { + if _, ok := crawled[dest.URL.String()]; ok { continue // TODO handle } // mark this page as complete - parsed[dest.URL.String()] = dest + crawled[dest.URL.String()] = dest // fetch page text, err := fetchPage(dest) @@ -64,14 +40,21 @@ func docrawl(url string) []*Page { continue // TODO handle } - // parse it + // parse links _, links, err := parseLinks(text) if err != nil { continue // TODO handle } for _, l := range links { - u, _ := resolveURL(dest.URL.String(), l) // TODO check error - dest.Children = append(dest.Children, &Page{URL: u, Base: dest}) + p, _ := NewPage(dest, l) // TODO check error + dest.Children = append(dest.Children, p) + + // see if we should queue this new page for crawling + _, haveCrawled := crawled[p.URL.String()] + _, haveQueued := queued[p.URL.String()] + if !haveCrawled && !haveQueued { + queued[p.URL.String()] = p + } } // add page to list diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 1641c06..e99935a 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -78,14 +78,14 @@ func TestSimpleFetch(t *testing.T) { func TestSimpleMap(t *testing.T) { pages := docrawl(baseURL) t.Log("pages = ", pages) - if len(pages) != 1 { + if len(pages) != 4 { t.Logf("got %v, wanted %v\n", len(pages), 1) t.Fatal("got wrong number of pages") } if pages[0].URL.String() != baseURL { t.Error("page URL is invalid") } - if len(pages[0].Children) != 2 { + if len(pages[0].Children) != 3 { t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1) t.Fatal("got wrong number of links") } @@ -99,6 +99,11 @@ func TestSimpleMap(t *testing.T) { t.Logf("Wanted: %q\n", baseURL+"assets/image.png") t.Error("link name is incorrect") } + if pages[0].Children[2].URL.String() != baseURL+"scripts/blah.js" { + t.Logf(" Got: %q\n", pages[0].Children[2].URL.String()) + t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") + t.Error("link name is incorrect") + } } // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results diff --git a/src/docrawler/page.go b/src/docrawler/page.go new file mode 100644 index 0000000..37a7f7c --- /dev/null +++ b/src/docrawler/page.go @@ -0,0 +1,32 @@ +package main + +import ( + "net/url" +) + +// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type Page struct { + URL *url.URL + Base *Page + Title string + MediaType string + Children []*Page +} + +// NewPage takes a referring Page + a URL and returns a new &Page{} +func NewPage(referrer *Page, rawurl string) (*Page, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.URL.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // create struct and return + return &Page{URL: u}, nil +} diff --git a/src/docrawler/testsite/index.html b/src/docrawler/testsite/index.html index 3a41cf9..d75978e 100644 --- a/src/docrawler/testsite/index.html +++ b/src/docrawler/testsite/index.html @@ -3,6 +3,7 @@
-
+ fooey!
+