diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 48de706..70e8702 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -5,23 +5,27 @@ import ( "os" ) +type itemSlice []*httpItem +type itemMap map[string]*httpItem + + // docrawl begins crawling the site at "url" -func docrawl(url string) []*Page { +func docrawl(url string) itemSlice { // our result structure - var sitemap []*Page + var sitemap itemSlice // what we've already parsed, what we still need to parse - crawled := make(map[string]*Page) - queued := make(map[string]*Page) + crawled := make(itemMap) + queued := make(itemMap) // create first page and add to the queue - homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.URL.String()] = homepage + homepage, _ := newHTTPItem(nil, url) // TODO check error + queued[homepage.url.String()] = homepage // main loop for len(queued) > 0 { // pop a page off of queued - var dest *Page + var dest *httpItem var destURL string for destURL, dest = range queued { break @@ -29,52 +33,52 @@ func docrawl(url string) []*Page { delete(queued, destURL) // mark this page as complete - crawled[dest.URL.String()] = dest + crawled[dest.url.String()] = dest // add page to list sitemap = append(sitemap, dest) // make sure this URL has the same hostname as our first page - if dest.URL.Host != homepage.URL.Host { + if dest.url.Host != homepage.url.Host { // skip URLs associated with other Hosts - dest.Skipped = true + dest.skipped = true continue } // fetch page text, err := fetchPage(dest) if err != nil { - dest.Broken = true + dest.broken = true continue } // parse links title, links, err := parseLinks(text) if err != nil { - dest.Broken = true + dest.broken = true continue } - dest.Title = title + dest.title = title for _, l := range links { - p, err := NewPage(dest, l) + p, err := newHTTPItem(dest, l) if err != nil { continue // TODO check error } // see if we already know about this page - pCrawled, haveCrawled := crawled[p.URL.String()] - pQueued, haveQueued := queued[p.URL.String()] + pCrawled, haveCrawled := crawled[p.url.String()] + pQueued, haveQueued := queued[p.url.String()] if haveCrawled { // add the previously crawled page to children - dest.Children = append(dest.Children, pCrawled) + dest.children = append(dest.children, pCrawled) } else if haveQueued { // add the previously queued page to children - dest.Children = append(dest.Children, pQueued) + dest.children = append(dest.children, pQueued) } else { // add this new page (never seen) to children - dest.Children = append(dest.Children, p) + dest.children = append(dest.children, p) // .. and queue it up for crawling - queued[p.URL.String()] = p + queued[p.url.String()] = p } } } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6233cac..abd294f 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - page, err := NewPage(nil, baseURL+"fetch_test.html") + page, err := newHTTPItem(nil, baseURL+"fetch_test.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } body, err := fetchPage(page) if err != nil { @@ -82,19 +82,19 @@ func TestSimpleMap(t *testing.T) { // cycle through every page, counting stuff, finding specific pages // so that we can then verify certain aspects of each page skipped, children := 0, 0 - var homepage, aboutpage, skippage *Page + var homepage, aboutpage, skippage *httpItem for _, p := range pages { - if p.Title == "Home" { + if p.title == "Home" { homepage = p - } else if p.Title == "About Test" { + } else if p.title == "About Test" { aboutpage = p - } else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" { + } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.Skipped { + if p.skipped { skipped++ } - children += len(p.Children) + children += len(p.children) } // check page counts @@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) { } // check first page URL - if homepage.URL.String() != baseURL { + if homepage.url.String() != baseURL { t.Error("page URL is invalid") } // specific page number of children - if len(homepage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(homepage.Children), 4) + if len(homepage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } - if len(aboutpage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4) + if len(aboutpage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4) t.Fatal("got wrong number of links") } // make sure remote pages were skipped - if homepage.Skipped != false { - t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String()) + if homepage.skipped != false { + t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.Skipped != true { - t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String()) + if skippage.skipped != true { + t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } // verify page content // TODO find a better way to do this - //if homepage.Children[0].URL.String() != baseURL+"about.html" { - //t.Logf(" Got: %q\n", homepage.Children[0].URL.String()) + //if homepage.children[0].url.String() != baseURL+"about.html" { + //t.Logf(" Got: %q\n", homepage.children[0].url.String()) //t.Logf("Wanted: %q\n", baseURL+"about.html") //t.Error("link name is incorrect") //} - //if homepage.Children[1].URL.String() != baseURL+"assets/image.png" { - //t.Logf(" Got: %q\n", homepage.Children[1].URL.String()) + //if homepage.children[1].url.String() != baseURL+"assets/image.png" { + //t.Logf(" Got: %q\n", homepage.children[1].url.String()) //t.Logf("Wanted: %q\n", baseURL+"assets/image.png") //t.Error("link name is incorrect") //} - //if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" { - //t.Logf(" Got: %q\n", homepage.Children[3].URL.String()) + //if homepage.children[3].url.String() != baseURL+"scripts/blah.js" { + //t.Logf(" Got: %q\n", homepage.children[3].url.String()) //t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") //t.Error("link name is incorrect") //} @@ -157,28 +157,28 @@ func TestSimpleMap(t *testing.T) { // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - basepage, err := NewPage(nil, baseURL) + basepage, err := newHTTPItem(nil, baseURL) if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" { t.Error("problem fetching filetype") } - page, err := NewPage(basepage, "about.html") + page, err := newHTTPItem(basepage, "about.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" { t.Error("problem fetching filetype") } - page, err = NewPage(basepage, "assets/image.png") + page, err = newHTTPItem(basepage, "assets/image.png") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { - t.Logf("got %q, wanted %q", page.MediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" { + t.Logf("got %q, wanted %q", page.mediaType, "image/png") t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 9d41dc9..c7c83d9 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -13,9 +13,9 @@ var ( ) // fetchFiletype performs an http HEAD to get the media type, and sets it -// directly in Page.MediaType -func fetchFiletype(page *Page) error { - resp, err := http.Head(page.URL.String()) +// directly in httpItem.mediaType +func fetchFiletype(page *httpItem) error { + resp, err := http.Head(page.url.String()) if err != nil { return err } @@ -39,12 +39,12 @@ func fetchFiletype(page *Page) error { } // set and return - page.MediaType = mediatype + page.mediaType = mediatype return nil } -// fetchPage takes a Page, GETs it, and returns the body as a string -func fetchPage(page *Page) (string, error) { +// fetchPage takes a httpItem, GETs it, and returns the body as a string +func fetchPage(page *httpItem) (string, error) { // figure out the file type err := fetchFiletype(page) if err != nil { @@ -52,12 +52,12 @@ func fetchPage(page *Page) (string, error) { } // we only want to fetch html - if page.MediaType != "text/html" { + if page.mediaType != "text/html" { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.URL.String()) + resp, err := http.Get(page.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index 7cdff84..0cf42d1 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -6,7 +6,7 @@ import ( // TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedHeaderFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } @@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) { // TestFailedFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 87f5ad1..5b26247 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,32 +15,32 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*Page) []*Location { +func sitemapToLocations(pages []*httpItem) []*Location { // build a map of our pages - pageMap := make(map[string]*Page) + pageMap := make(map[string]*httpItem) for _, p := range pages { - pageMap[p.URL.String()] = p + pageMap[p.url.String()] = p } // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.MediaType == "text/html" { + if p.mediaType == "text/html" { // create a location for this page - l := &Location{URL: p.URL.String(), Title: p.Title} + l := &Location{URL: p.url.String(), Title: p.title} // add its children - for _, c := range p.Children { + for _, c := range p.children { // look up this child's media type from the root list of pages - //mediaType := pageMap[c.URL.String()].MediaType - if c.Skipped { - l.Remote = append(l.Remote, c.URL.String()) - } else if c.MediaType == "text/html" { - l.Links = append(l.Links, c.URL.String()) - } else if c.Broken { - l.Broken = append(l.Broken, c.URL.String()) + //mediaType := pageMap[c.url.String()].mediaType + if c.skipped { + l.Remote = append(l.Remote, c.url.String()) + } else if c.mediaType == "text/html" { + l.Links = append(l.Links, c.url.String()) + } else if c.broken { + l.Broken = append(l.Broken, c.url.String()) } else { - l.Assets = append(l.Assets, c.URL.String()) + l.Assets = append(l.Assets, c.url.String()) } } diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 9e8be32..d6f8ac8 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -4,22 +4,22 @@ import ( "net/url" ) -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page +// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type httpItem struct { + url *url.URL + title string + mediaType string + skipped bool + broken bool + children []*httpItem } -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { +// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} +func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { // determine the base URL so we can resulve this into a full URL baseURL := "" if referrer != nil { - baseURL = referrer.URL.String() + baseURL = referrer.url.String() } // resolve URL into a full absolute URL (non-relative) @@ -29,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) { } // create struct and return - return &Page{URL: u, Skipped: false, Broken: false}, nil + return &httpItem{url: u, skipped: false, broken: false}, nil } diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go index 217f290..d9a2143 100644 --- a/src/docrawler/page_test.go +++ b/src/docrawler/page_test.go @@ -4,10 +4,10 @@ import ( "testing" ) -// TestBadNewPage tests NewPage to see if we are getting expected failures -func TestBadNewPage(t *testing.T) { - _, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") +// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures +func TestBadnewHTTPItem(t *testing.T) { + _, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") if err == nil { - t.Error("tried creating a NewPage with an invalid URL but didn't get an error") + t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error") } }