diff --git a/docs/TODO.md b/docs/TODO.md index 450c69e..5fe18f4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -6,10 +6,10 @@ * should only crawl these pages once * but should be included in the links part * when sorting links, "uniq" them as well +* be smart about index.html? ## Cleanup -* rename Page -> crawlerState * helper function to do the whole crawl process * document code better * test with godoc @@ -17,7 +17,6 @@ ## Features -* some sort of status during crawl? * command line options, flag parsing * check RFCs on URL character sets, compliance, etc. * packagize parts of app diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 08ac36f..af538a3 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,84 +2,120 @@ package main import ( "fmt" + "log" "os" + "time" ) -// docrawl begins crawling the site at "url" -func docrawl(url string) []*Page { - // our result structure - var sitemap []*Page +// docrawl begins crawling the site at "homeurl" +func docrawl(homeurl string) itemSlice { + // set of what we have already crawled + crawled := make(map[string]struct{}) - // what we've already parsed, what we still need to parse - crawled := make(map[string]*Page) - queued := make(map[string]*Page) + // map (of urlString -> httpItem) of our results + results := make(itemMap) - // create first page and add to the queue - homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.Identifier] = homepage + // a channel to receive our results on + rchan := make(chan *httpItem) - // main loop - for len(queued) > 0 { - // pop a page off of queued - var dest *Page - var destURL string - for destURL, dest = range queued { - break - } - delete(queued, destURL) + // start a ticker which we'll use to output status and check for completion + ticker := time.Tick(250 * time.Millisecond) - // mark this page as complete - crawled[dest.Identifier] = dest + // create first page's httpItem + homeitem, err := newHTTPItem(nil, homeurl) + if err != nil { + log.Fatal("unable to create httpItem for homeurl") + } - // add page to list - sitemap = append(sitemap, dest) + // set our number of outstanding pages (initially 1 to account for first page) + crawlingCount := 1 - // make sure this URL has the same hostname as our first page - if dest.URL.Host != homepage.URL.Host { - // skip URLs associated with other Hosts - dest.Skipped = true - continue - } + // start the home page crawl + crawled[homeitem.url.String()] = struct{}{} + go crawlItem(homeitem, rchan) - // fetch page - text, err := fetchPage(dest) - if err != nil { - dest.Broken = true - continue - } + // wait for results + for { + select { + case r := <-rchan: // new results? + // add result to our results map + results[r.url.String()] = r - // parse links - title, links, err := parseLinks(text) - if err != nil { - dest.Broken = true - continue - } - dest.Title = title - for _, l := range links { - p, err := NewPage(dest, l) - if err != nil { - continue // TODO check error + // decrease the outstanding page count by 1 + crawlingCount-- + + // start crawly any new child pages we haven't yet crawled + for i, c := range r.children { + // see if we already have a result for this page, if so, point to that result + if existing, ok := results[c.url.String()]; ok { + r.children[i] = existing + continue + } + + // see if we're already crawling this page (but maybe don't have results yet) + if _, ok := crawled[c.url.String()]; !ok { + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = struct{}{} + go crawlItem(c, rchan) + } } - // see if we already know about this page - pCrawled, haveCrawled := crawled[p.Identifier] - pQueued, haveQueued := queued[p.Identifier] - if haveCrawled { - // add the previously crawled page to children - dest.Children = append(dest.Children, pCrawled) - } else if haveQueued { - // add the previously queued page to children - dest.Children = append(dest.Children, pQueued) - } else { - // add this new page (never seen) to children - dest.Children = append(dest.Children, p) - // .. and queue it up for crawling - queued[p.Identifier] = p + case <-ticker: // our regular ticker. for status output and checking for completion. + // output status to console + log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount) + + // see if we're finished + if crawlingCount == 0 { + // finished! convert results map to a slice and return it + rslice := itemSlice{} + for _, v := range results { + rslice = append(rslice, v) + } + return rslice } } } +} - return sitemap +// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, +// and filling out its structure as much as possible +func crawlItem(item *httpItem, rchan chan<- *httpItem) { + // make sure this item is the same domain (i.e. URL "host part") as its referrer + if item.refurl != nil && item.url.Host != item.refurl.Host { + // skip URLs associated with other Hosts + item.linkType = tRemote + rchan <- item + return + } + + // fetch page + text, err := fetchPage(item) + if err != nil { + rchan <- item + return + } + + // parse links + title, links, _ := parseLinks(text) // TODO remove error from parseLinks, not needed + if err != nil { + item.linkType = tBroken + rchan <- item + return + } + item.title = title + + // walk links and add them as children to the current item + for _, l := range links { + newItem, err := newHTTPItem(item, l) + if err != nil { + continue // TODO bad item + } + item.children = append(item.children, newItem) + } + + // send back our item struct now that it's all filled out + rchan <- item } func main() { diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6233cac..6be6209 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - page, err := NewPage(nil, baseURL+"fetch_test.html") + page, err := newHTTPItem(nil, baseURL+"fetch_test.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } body, err := fetchPage(page) if err != nil { @@ -82,19 +82,19 @@ func TestSimpleMap(t *testing.T) { // cycle through every page, counting stuff, finding specific pages // so that we can then verify certain aspects of each page skipped, children := 0, 0 - var homepage, aboutpage, skippage *Page + var homepage, aboutpage, skippage *httpItem for _, p := range pages { - if p.Title == "Home" { + if p.title == "Home" { homepage = p - } else if p.Title == "About Test" { + } else if p.title == "About Test" { aboutpage = p - } else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" { + } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.Skipped { + if p.linkType == tRemote { skipped++ } - children += len(p.Children) + children += len(p.children) } // check page counts @@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) { } // check first page URL - if homepage.URL.String() != baseURL { + if homepage.url.String() != baseURL { t.Error("page URL is invalid") } // specific page number of children - if len(homepage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(homepage.Children), 4) + if len(homepage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } - if len(aboutpage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4) + if len(aboutpage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4) t.Fatal("got wrong number of links") } // make sure remote pages were skipped - if homepage.Skipped != false { - t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String()) + if homepage.linkType == tRemote { + t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.Skipped != true { - t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String()) + if skippage.linkType != tRemote { + t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } // verify page content // TODO find a better way to do this - //if homepage.Children[0].URL.String() != baseURL+"about.html" { - //t.Logf(" Got: %q\n", homepage.Children[0].URL.String()) + //if homepage.children[0].url.String() != baseURL+"about.html" { + //t.Logf(" Got: %q\n", homepage.children[0].url.String()) //t.Logf("Wanted: %q\n", baseURL+"about.html") //t.Error("link name is incorrect") //} - //if homepage.Children[1].URL.String() != baseURL+"assets/image.png" { - //t.Logf(" Got: %q\n", homepage.Children[1].URL.String()) + //if homepage.children[1].url.String() != baseURL+"assets/image.png" { + //t.Logf(" Got: %q\n", homepage.children[1].url.String()) //t.Logf("Wanted: %q\n", baseURL+"assets/image.png") //t.Error("link name is incorrect") //} - //if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" { - //t.Logf(" Got: %q\n", homepage.Children[3].URL.String()) + //if homepage.children[3].url.String() != baseURL+"scripts/blah.js" { + //t.Logf(" Got: %q\n", homepage.children[3].url.String()) //t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") //t.Error("link name is incorrect") //} @@ -157,28 +157,28 @@ func TestSimpleMap(t *testing.T) { // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - basepage, err := NewPage(nil, baseURL) + basepage, err := newHTTPItem(nil, baseURL) if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage { t.Error("problem fetching filetype") } - page, err := NewPage(basepage, "about.html") + page, err := newHTTPItem(basepage, "about.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage { t.Error("problem fetching filetype") } - page, err = NewPage(basepage, "assets/image.png") + page, err = newHTTPItem(basepage, "assets/image.png") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { - t.Logf("got %q, wanted %q", page.MediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.linkType != tAsset { + t.Logf("got %q, wanted %q", page.linkType, tAsset) t.Error("problem fetching filetype") } } diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 9d41dc9..9cd67a6 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -9,19 +9,22 @@ import ( var ( errContentTypeNotFound = errors.New("no Content-Type header found") - errFetchError = errors.New("couldn't fetch page") + errFetchError = errors.New("couldn't fetch item") + errFileTypeUnknown = errors.New("couldn't determine file type") ) // fetchFiletype performs an http HEAD to get the media type, and sets it -// directly in Page.MediaType -func fetchFiletype(page *Page) error { - resp, err := http.Head(page.URL.String()) +// directly in httpItem.mediaType +func fetchFiletype(item *httpItem) error { + resp, err := http.Head(item.url.String()) if err != nil { + item.linkType = tBroken return err } // check response code if resp.StatusCode != http.StatusOK { + item.linkType = tBroken return errFetchError } @@ -29,35 +32,46 @@ func fetchFiletype(page *Page) error { contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found + item.linkType = tBroken return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) if err != nil { + item.linkType = tBroken return err } - // set and return - page.MediaType = mediatype + // success, set item type and return + if mediatype == "text/html" { + item.linkType = tHTMLPage + } else { + item.linkType = tAsset + } return nil } -// fetchPage takes a Page, GETs it, and returns the body as a string -func fetchPage(page *Page) (string, error) { +// fetchPage takes a httpItem, GETs it, and returns the body as a string +func fetchPage(item *httpItem) (string, error) { // figure out the file type - err := fetchFiletype(page) + err := fetchFiletype(item) if err != nil { return "", err } - // we only want to fetch html - if page.MediaType != "text/html" { + // make sure we got a file type! + if item.linkType == tUnknown { + return "", errFileTypeUnknown + } + + // we only want to fetch html, so return if it's anything else + if item.linkType != tHTMLPage { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.URL.String()) + resp, err := http.Get(item.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index 7cdff84..0cf42d1 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -6,7 +6,7 @@ import ( // TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedHeaderFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } @@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) { // TestFailedFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } diff --git a/src/docrawler/item.go b/src/docrawler/item.go new file mode 100644 index 0000000..cb9d90c --- /dev/null +++ b/src/docrawler/item.go @@ -0,0 +1,55 @@ +package main + +import ( + "net/url" +) + +// itemSlice is a convenience type for a slice of items +type itemSlice []*httpItem + +// itemMap convenience type for map of string's (URL) to items +type itemMap map[string]*httpItem + +// itemType is an enum so we know how to classify each item +type itemType int + +const ( + tUnknown itemType = iota + tHTMLPage + tAsset + tRemote + tBroken +) + +// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type httpItem struct { + url *url.URL + refurl *url.URL + title string + linkType itemType + children itemSlice +} + +// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} +func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.url.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // figure out referrer url + var rurl *url.URL + if referrer != nil { + rurl = referrer.url + } + + // create struct and return + return &httpItem{url: u, refurl: rurl}, nil +} diff --git a/src/docrawler/item_test.go b/src/docrawler/item_test.go new file mode 100644 index 0000000..d9a2143 --- /dev/null +++ b/src/docrawler/item_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" +) + +// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures +func TestBadnewHTTPItem(t *testing.T) { + _, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") + if err == nil { + t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error") + } +} diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 2b6b959..426599b 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,32 +15,41 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*Page) []*Location { +// implement Location slice sorting (by URL) +type byURL []*Location + +// implement Sort interface on byURL (which is a []*Location) +func (l byURL) Len() int { return len(l) } +func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL } +func (l byURL) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling +func sitemapToLocations(pages itemSlice) []*Location { // build a map of our pages - pageMap := make(map[string]*Page) + pageMap := make(map[string]*httpItem) for _, p := range pages { - pageMap[p.URL.String()] = p + pageMap[p.url.String()] = p } // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.MediaType == "text/html" { + if p.linkType == tHTMLPage { // create a location for this page - l := &Location{URL: p.URL.String(), Title: p.Title} + l := &Location{URL: p.url.String(), Title: p.title} // add its children - for _, c := range p.Children { + for _, c := range p.children { // look up this child's media type from the root list of pages - //mediaType := pageMap[c.URL.String()].MediaType - if c.Skipped { - l.Remote = append(l.Remote, c.URL.String()) - } else if c.MediaType == "text/html" { - l.Links = append(l.Links, c.URL.String()) - } else if c.Broken { - l.Broken = append(l.Broken, c.URL.String()) + //mediaType := pageMap[c.url.String()].mediaType + if c.linkType == tRemote { + l.Remote = append(l.Remote, c.url.String()) + } else if c.linkType == tHTMLPage { + l.Links = append(l.Links, c.url.String()) + } else if c.linkType == tBroken { + l.Broken = append(l.Broken, c.url.String()) } else { - l.Assets = append(l.Assets, c.URL.String()) + l.Assets = append(l.Assets, c.url.String()) } } @@ -59,6 +68,8 @@ func sitemapToLocations(pages []*Page) []*Location { } } + // sort the Locations themselves and return + sort.Sort(byURL(locations)) return locations } diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go deleted file mode 100644 index 217f290..0000000 --- a/src/docrawler/page_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "testing" -) - -// TestBadNewPage tests NewPage to see if we are getting expected failures -func TestBadNewPage(t *testing.T) { - _, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") - if err == nil { - t.Error("tried creating a NewPage with an invalid URL but didn't get an error") - } -}