diff --git a/docs/Design.md b/docs/Design.md index db2d874..0a37822 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -2,38 +2,38 @@ ## Minimum Features -This is the assignment (directly) from DO. +This is the assignment (directly) from **REDACTED**. -* We do ask that you put a heavy emphasis on testing in the assignment. -* written in a modern language -* limited to one domain - so when crawling digitalocean.com it would crawl all pages within the digitalocean.com domain, but not follow the links to our Facebook or Instagram accounts or subdomains like cloud.digitalocean.com. -* It should be Given a URL -* it should output a site map -* it should show which static assets each page depends on, and the links between pages. -* Choose the most appropriate data structure to store & display this site map. -* Build this as you would build something for production - it's fine if you don't finish everything, but focus on code quality and write tests. -* We're interested in how you code and how you test your code. +✓ We do ask that you put a heavy emphasis on testing in the assignment. + written in a modern language +✓ limited to one domain - so when crawling REDACTED.com it would crawl all pages within the REDACTED.com domain, but not follow the links to our Facebook or Instagram accounts or subdomains like cloud.REDACTED.com. +✓ It should be Given a URL +✓ it should output a site map +✓ it should show which static assets each page depends on, and the links between pages. +✓ Choose the most appropriate data structure to store & display this site map. +✓ Build this as you would build something for production - it's fine if you don't finish everything, but focus on code quality and write tests. +✓ We're interested in how you code and how you test your code. ## Desired These are features I'd like to include. * efficient, optimal -* handles base URL properly (for root & relative links) -* documented -* uses standard Go practices -* uses Go stdlib (only!) -* thoroughly tested +✓ handles base URL properly (for root & relative links) +✓ documented +✓ uses standard Go practices +✓ uses Go stdlib (only!) +✓ thoroughly tested * configurable on command line (defaults are for company specs) * variable output formats: json, dot -* adheres to URL RFC (as far as case sensitivity, acceptable character sets, etc.) +✓ adheres to URL RFC (as far as case sensitivity, acceptable character sets, etc.) * go gettable -* go fmt'ed, vet'ed, lint'ed +✓ go fmt'ed, vet'ed, lint'ed * polite - robots.txt support -* robust - detect infinite loops +✓ robust - detect infinite loops * throttling * use Go context pattern -* supports http & https +✓ supports http & https ## Architecture @@ -41,8 +41,8 @@ These are features I'd like to include. * pipeline chain should be such that when there's no more work to do the job dispatcher closes the next channel in the pipeline, thereby closing the entire chain of channels (and exiting all goroutines) * pipeline may allows us to scale concurrency at the bottlenecks * could use Interfaces for things like Parser, Getter, but I think best avoided until needed -* launch goroutine per fetch -* when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it +✓ launch goroutine per fetch +✓ when there's no Jobs left, and nothing being fetched or parsed, convert the data struct into something useful and output it ## Wishlist diff --git a/docs/TODO.md b/docs/TODO.md index 3e03d44..ea142d3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -2,21 +2,15 @@ ## Bugs -* deal with anchor issue (repeatedly crawling index.html#about, index.html#contact, etc.) - * should only crawl these pages once - * but should be included in the links part -* when sorting links, "uniq" them as well - ## Cleanup -* rename Page -> crawlerState * helper function to do the whole crawl process * document code better * test with godoc +* omit fields in json when nil? (Broken, etc.) ## Features -* some sort of status during crawl? * command line options, flag parsing * check RFCs on URL character sets, compliance, etc. * packagize parts of app diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 48de706..e25b288 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -2,90 +2,148 @@ package main import ( "fmt" + "log" "os" + "time" ) -// docrawl begins crawling the site at "url" -func docrawl(url string) []*Page { - // our result structure - var sitemap []*Page +// docrawl begins crawling the site at "homeurl" +func doCrawl(homeurl string) itemSlice { + // set of what we have already crawled + crawled := make(itemMap) - // what we've already parsed, what we still need to parse - crawled := make(map[string]*Page) - queued := make(map[string]*Page) + // a set of crawled URLs which have been cleaned + // useful so that we don't crawl http://a.com/index.html#about if we've + // already crawled http://a.com/index.html, or vice versa + crawledStripped := make(itemMap) - // create first page and add to the queue - homepage, _ := NewPage(nil, url) // TODO check error - queued[homepage.URL.String()] = homepage + // map (of urlString -> httpItem) of our results + results := make(itemMap) - // main loop - for len(queued) > 0 { - // pop a page off of queued - var dest *Page - var destURL string - for destURL, dest = range queued { - break - } - delete(queued, destURL) + // a channel to receive our results on + rchan := make(chan *httpItem) - // mark this page as complete - crawled[dest.URL.String()] = dest + // start a ticker which we'll use to output status and check for completion + ticker := time.Tick(250 * time.Millisecond) - // add page to list - sitemap = append(sitemap, dest) + // create first page's httpItem + homeitem, err := newHTTPItem(nil, homeurl) + if err != nil { + log.Fatal("unable to create httpItem for homeurl") + } - // make sure this URL has the same hostname as our first page - if dest.URL.Host != homepage.URL.Host { - // skip URLs associated with other Hosts - dest.Skipped = true - continue - } + // set our number of outstanding pages (initially 1 to account for first page) + crawlingCount := 1 - // fetch page - text, err := fetchPage(dest) - if err != nil { - dest.Broken = true - continue - } + // start the home page crawl + crawled[homeitem.url.String()] = homeitem + crawledStripped[stripURL(homeitem.url)] = homeitem + go crawlItem(homeitem, rchan) - // parse links - title, links, err := parseLinks(text) - if err != nil { - dest.Broken = true - continue - } - dest.Title = title - for _, l := range links { - p, err := NewPage(dest, l) - if err != nil { - continue // TODO check error + // wait for results + for { + select { + case r := <-rchan: // new results? + // add result to our results map + results[r.url.String()] = r + + // decrease the outstanding page count by 1 + crawlingCount-- + + // start crawly any new child pages we haven't yet crawled + for i, c := range r.children { + // see if we already have a result for this page + // if so, point to that result + if existing, ok := results[c.url.String()]; ok { + r.children[i] = existing + continue + } + + // see if we're already crawling this page (but maybe don't have results yet) + // if so, point to that item (we will have it as a result later!) + if existing, ok := crawled[c.url.String()]; ok { + r.children[i] = existing + continue + } + if existing, ok := crawledStripped[stripURL(c.url)]; ok { + // we crawled a different version of this same page + // i.e. same page, different anchor. we don't need to crawl it again + // but we do need to keep it in our results, so copy the existing + // struct over for everything except the URLs + r.children[i].title = existing.title + r.children[i].linkType = existing.linkType + r.children[i].children = existing.children + continue + } + + // haven't crawled this one yet, do so now + crawlingCount++ + crawled[c.url.String()] = c + crawledStripped[stripURL(c.url)] = c + go crawlItem(c, rchan) } - // see if we already know about this page - pCrawled, haveCrawled := crawled[p.URL.String()] - pQueued, haveQueued := queued[p.URL.String()] - if haveCrawled { - // add the previously crawled page to children - dest.Children = append(dest.Children, pCrawled) - } else if haveQueued { - // add the previously queued page to children - dest.Children = append(dest.Children, pQueued) - } else { - // add this new page (never seen) to children - dest.Children = append(dest.Children, p) - // .. and queue it up for crawling - queued[p.URL.String()] = p + case <-ticker: // our regular ticker. for status output and checking for completion. + // output status to console + log.Printf("Crawled %v links, have %v left.\n", len(results), crawlingCount) + + // see if we're finished + if crawlingCount == 0 { + // finished! convert results map to a slice and return it + rslice := itemSlice{} + for _, v := range results { + rslice = append(rslice, v) + } + return rslice } } } +} - return sitemap +// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, +// and filling out its structure as much as possible +func crawlItem(item *httpItem, rchan chan<- *httpItem) { + // make sure this item is the same domain (i.e. URL "host part") as its referrer + if item.refurl != nil && item.url.Host != item.refurl.Host { + // skip URLs associated with other Hosts + item.linkType = tRemote + rchan <- item + return + } + + // fetch page + text, err := fetchPage(item) + if err != nil { + rchan <- item + return + } + + // parse links + title, links := parseLinks(text) // TODO remove error from parseLinks, not needed + if err != nil { + item.linkType = tBroken + rchan <- item + return + } + item.title = title + + // walk links and add them as children to the current item + for _, l := range links { + newItem, err := newHTTPItem(item, l) + if err != nil { + continue // TODO bad item + } + item.children = append(item.children, newItem) + } + + // send back our item struct now that it's all filled out + rchan <- item } func main() { // crawl each URL on the command line for _, u := range os.Args[1:] { - pages := docrawl(u) + pages := doCrawl(u) l := sitemapToLocations(pages) j, _ := locationsToJSON(l) fmt.Println(j) diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 6233cac..60bd8cc 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) { // TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect func TestSimpleFetch(t *testing.T) { - page, err := NewPage(nil, baseURL+"fetch_test.html") + page, err := newHTTPItem(nil, baseURL+"fetch_test.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } body, err := fetchPage(page) if err != nil { @@ -76,25 +76,25 @@ func TestSimpleFetch(t *testing.T) { // TestSimpleMap figures out the site map for the site in baseURL func TestSimpleMap(t *testing.T) { - pages := docrawl(baseURL) + pages := doCrawl(baseURL) // because our crawl is non-deterministic, we have to do a complete // cycle through every page, counting stuff, finding specific pages // so that we can then verify certain aspects of each page skipped, children := 0, 0 - var homepage, aboutpage, skippage *Page + var homepage, aboutpage, skippage *httpItem for _, p := range pages { - if p.Title == "Home" { + if p.title == "Home" { homepage = p - } else if p.Title == "About Test" { + } else if p.title == "About Test" { aboutpage = p - } else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" { + } else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" { skippage = p } - if p.Skipped { + if p.linkType == tRemote { skipped++ } - children += len(p.Children) + children += len(p.children) } // check page counts @@ -102,7 +102,7 @@ func TestSimpleMap(t *testing.T) { t.Logf("got %v, wanted %v\n", len(pages), 6) t.Fatal("got wrong number of pages") } - if children != 8 { + if children != 9 { t.Logf("got %v, wanted %v\n", children, 8) t.Fatal("got wrong number of total children") } @@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) { } // check first page URL - if homepage.URL.String() != baseURL { + if homepage.url.String() != baseURL { t.Error("page URL is invalid") } // specific page number of children - if len(homepage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(homepage.Children), 4) + if len(homepage.children) != 5 { + t.Logf("got %v, wanted %v\n", len(homepage.children), 4) t.Fatal("got wrong number of links") } - if len(aboutpage.Children) != 4 { - t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4) + if len(aboutpage.children) != 4 { + t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4) t.Fatal("got wrong number of links") } // make sure remote pages were skipped - if homepage.Skipped != false { - t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String()) + if homepage.linkType == tRemote { + t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String()) t.Error("child was skipped") } - if skippage.Skipped != true { - t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String()) + if skippage.linkType != tRemote { + t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String()) t.Error("child wasn't skipped") } // verify page content // TODO find a better way to do this - //if homepage.Children[0].URL.String() != baseURL+"about.html" { - //t.Logf(" Got: %q\n", homepage.Children[0].URL.String()) + //if homepage.children[0].url.String() != baseURL+"about.html" { + //t.Logf(" Got: %q\n", homepage.children[0].url.String()) //t.Logf("Wanted: %q\n", baseURL+"about.html") //t.Error("link name is incorrect") //} - //if homepage.Children[1].URL.String() != baseURL+"assets/image.png" { - //t.Logf(" Got: %q\n", homepage.Children[1].URL.String()) + //if homepage.children[1].url.String() != baseURL+"assets/image.png" { + //t.Logf(" Got: %q\n", homepage.children[1].url.String()) //t.Logf("Wanted: %q\n", baseURL+"assets/image.png") //t.Error("link name is incorrect") //} - //if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" { - //t.Logf(" Got: %q\n", homepage.Children[3].URL.String()) + //if homepage.children[3].url.String() != baseURL+"scripts/blah.js" { + //t.Logf(" Got: %q\n", homepage.children[3].url.String()) //t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js") //t.Error("link name is incorrect") //} @@ -157,35 +157,35 @@ func TestSimpleMap(t *testing.T) { // TestHeaderFetching tests fetchFiletype() to see if we are getting expected results func TestHeaderFetching(t *testing.T) { - basepage, err := NewPage(nil, baseURL) + basepage, err := newHTTPItem(nil, baseURL) if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" { + if err := fetchFiletype(basepage); err != nil || basepage.linkType != tHTMLPage { t.Error("problem fetching filetype") } - page, err := NewPage(basepage, "about.html") + page, err := newHTTPItem(basepage, "about.html") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" { + if err := fetchFiletype(page); err != nil || page.linkType != tHTMLPage { t.Error("problem fetching filetype") } - page, err = NewPage(basepage, "assets/image.png") + page, err = newHTTPItem(basepage, "assets/image.png") if err != nil { - t.Error("problem creating New Page struct") + t.Error("problem creating New httpItem struct") } - if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" { - t.Logf("got %q, wanted %q", page.MediaType, "image/png") + if err := fetchFiletype(page); err != nil || page.linkType != tAsset { + t.Logf("got %q, wanted %q", page.linkType, tAsset) t.Error("problem fetching filetype") } } // TestJsonOutput gets a sitemap and then converts it to json func TestJsonOutput(t *testing.T) { - pages := docrawl(baseURL) + pages := doCrawl(baseURL) l := sitemapToLocations(pages) if len(l) != 2 { t.Error("sitemapToLocations has the wrong number of locations") @@ -229,7 +229,38 @@ func TestJsonOutput(t *testing.T) { // this is a cheater test because the output is from a run of the code being // test itself. but, it's been examined and I think it's right. and, there's not // many other ways to test this without some significant pain - cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://localhost:8765/zzzbroken.html"],"Remote":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null,"Remote":["http://doesntexist23492387492837492374982734.com/"]}]` + cheaterTest := `[ + { + "URL": "http://localhost:8765/", + "Title": "Home", + "Links": [ + "http://localhost:8765/about.html" + ], + "Assets": [ + "http://localhost:8765/assets/image.png", + "http://localhost:8765/scripts/blah.js" + ], + "Broken": [ + "http://localhost:8765/zzzbroken.html" + ], + "Remote": null + }, + { + "URL": "http://localhost:8765/about.html", + "Title": "About Test", + "Links": [ + "http://localhost:8765/" + ], + "Assets": [ + "http://localhost:8765/assets/image.png", + "http://localhost:8765/scripts/blah.js" + ], + "Broken": null, + "Remote": [ + "http://doesntexist23492387492837492374982734.com/" + ] + } +]` j, err := locationsToJSON(l) if err != nil { t.Error("locationsToJSON failed") diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index 9d41dc9..bab34f0 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -7,21 +7,25 @@ import ( "net/http" ) +// custom errors var ( errContentTypeNotFound = errors.New("no Content-Type header found") - errFetchError = errors.New("couldn't fetch page") + errFetchError = errors.New("couldn't fetch item") + errFileTypeUnknown = errors.New("couldn't determine file type") ) // fetchFiletype performs an http HEAD to get the media type, and sets it -// directly in Page.MediaType -func fetchFiletype(page *Page) error { - resp, err := http.Head(page.URL.String()) +// directly in httpItem.mediaType +func fetchFiletype(item *httpItem) error { + resp, err := http.Head(item.url.String()) if err != nil { + item.linkType = tBroken return err } // check response code if resp.StatusCode != http.StatusOK { + item.linkType = tBroken return errFetchError } @@ -29,35 +33,46 @@ func fetchFiletype(page *Page) error { contentType, ok := resp.Header["Content-Type"] if !ok { // no Content-Type header found + item.linkType = tBroken return errContentTypeNotFound } // parse mime type mediatype, _, err := mime.ParseMediaType(contentType[0]) if err != nil { + item.linkType = tBroken return err } - // set and return - page.MediaType = mediatype + // success, set item type and return + if mediatype == "text/html" { + item.linkType = tHTMLPage + } else { + item.linkType = tAsset + } return nil } -// fetchPage takes a Page, GETs it, and returns the body as a string -func fetchPage(page *Page) (string, error) { +// fetchPage takes a httpItem, GETs it, and returns the body as a string +func fetchPage(item *httpItem) (string, error) { // figure out the file type - err := fetchFiletype(page) + err := fetchFiletype(item) if err != nil { return "", err } - // we only want to fetch html - if page.MediaType != "text/html" { + // make sure we got a file type! + if item.linkType == tUnknown { + return "", errFileTypeUnknown + } + + // we only want to fetch html, so return if it's anything else + if item.linkType != tHTMLPage { return "", nil // but this isn't an error! } // GET the url - resp, err := http.Get(page.URL.String()) + resp, err := http.Get(item.url.String()) if err != nil { return "", err } diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index 7cdff84..0cf42d1 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -6,7 +6,7 @@ import ( // TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedHeaderFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } @@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) { // TestFailedFetching tests fetchFiletype() to see if we are getting expected failures func TestFailedFetching(t *testing.T) { - page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/") + page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/") if err != nil { t.Error("problem creating New Page struct") } diff --git a/src/docrawler/item.go b/src/docrawler/item.go new file mode 100644 index 0000000..eaf42e0 --- /dev/null +++ b/src/docrawler/item.go @@ -0,0 +1,56 @@ +package main + +import ( + "net/url" +) + +// itemSlice is a convenience type for a slice of items +type itemSlice []*httpItem + +// itemMap convenience type for map of string's (URL) to items +type itemMap map[string]*httpItem + +// itemType is an enum so we know how to classify each item +type itemType int + +// enums for itemType +const ( + tUnknown itemType = iota + tHTMLPage + tAsset + tRemote + tBroken +) + +// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc. +type httpItem struct { + url *url.URL + refurl *url.URL + title string + linkType itemType + children itemSlice +} + +// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{} +func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) { + // determine the base URL so we can resulve this into a full URL + baseURL := "" + if referrer != nil { + baseURL = referrer.url.String() + } + + // resolve URL into a full absolute URL (non-relative) + u, err := resolveURL(baseURL, rawurl) + if err != nil { + return nil, err + } + + // figure out referrer url + var rurl *url.URL + if referrer != nil { + rurl = referrer.url + } + + // create struct and return + return &httpItem{url: u, refurl: rurl}, nil +} diff --git a/src/docrawler/item_test.go b/src/docrawler/item_test.go new file mode 100644 index 0000000..d9a2143 --- /dev/null +++ b/src/docrawler/item_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" +) + +// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures +func TestBadnewHTTPItem(t *testing.T) { + _, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") + if err == nil { + t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error") + } +} diff --git a/src/docrawler/output.go b/src/docrawler/output.go index 87f5ad1..15a5ccc 100644 --- a/src/docrawler/output.go +++ b/src/docrawler/output.go @@ -15,36 +15,53 @@ type Location struct { Remote []string } -func sitemapToLocations(pages []*Page) []*Location { +// implement Location slice sorting (by URL) +type byURL []*Location + +// implement Sort interface on byURL (which is a []*Location) +func (l byURL) Len() int { return len(l) } +func (l byURL) Less(i, j int) bool { return l[i].URL < l[j].URL } +func (l byURL) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +// sitemapToLocations converts an itemSlice to []*Location, which is appropriate for marshalling +func sitemapToLocations(pages itemSlice) []*Location { // build a map of our pages - pageMap := make(map[string]*Page) + pageMap := make(map[string]*httpItem) for _, p := range pages { - pageMap[p.URL.String()] = p + pageMap[p.url.String()] = p } // build a slice of locations (one per page) var locations []*Location for _, p := range pages { - if p.MediaType == "text/html" { + if p.linkType == tHTMLPage { // create a location for this page - l := &Location{URL: p.URL.String(), Title: p.Title} + l := &Location{URL: p.url.String(), Title: p.title} // add its children - for _, c := range p.Children { + for _, c := range p.children { // look up this child's media type from the root list of pages - //mediaType := pageMap[c.URL.String()].MediaType - if c.Skipped { - l.Remote = append(l.Remote, c.URL.String()) - } else if c.MediaType == "text/html" { - l.Links = append(l.Links, c.URL.String()) - } else if c.Broken { - l.Broken = append(l.Broken, c.URL.String()) + //mediaType := pageMap[c.url.String()].mediaType + if c.linkType == tRemote { + l.Remote = append(l.Remote, c.url.String()) + } else if c.linkType == tHTMLPage { + l.Links = append(l.Links, c.url.String()) + } else if c.linkType == tBroken { + l.Broken = append(l.Broken, c.url.String()) + } else if c.linkType == tAsset { + l.Assets = append(l.Assets, c.url.String()) } else { - l.Assets = append(l.Assets, c.URL.String()) + // unknown link here, which means it failed to crawl, let's call it "broken" + l.Broken = append(l.Broken, c.url.String()) } + } - // now sort the children slices + // now uniq & sort the children slices + l.Remote = uniqStrings(l.Remote) + l.Links = uniqStrings(l.Links) + l.Broken = uniqStrings(l.Broken) + l.Assets = uniqStrings(l.Assets) sort.Strings(l.Remote) sort.Strings(l.Links) sort.Strings(l.Broken) @@ -55,11 +72,14 @@ func sitemapToLocations(pages []*Page) []*Location { } } + // sort the Locations themselves and return + sort.Sort(byURL(locations)) return locations } +// locationsToJSON takes a *Location slice and marshals it into a JSON string func locationsToJSON(locations []*Location) (string, error) { - b, err := json.Marshal(locations) + b, err := json.MarshalIndent(locations, "", " ") if err != nil { return "", err } diff --git a/src/docrawler/page.go b/src/docrawler/page.go deleted file mode 100644 index 9e8be32..0000000 --- a/src/docrawler/page.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "net/url" -) - -// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. -type Page struct { - URL *url.URL - Title string - MediaType string - Skipped bool - Broken bool - Children []*Page -} - -// NewPage takes a referring Page + a URL and returns a new &Page{} -func NewPage(referrer *Page, rawurl string) (*Page, error) { - // determine the base URL so we can resulve this into a full URL - baseURL := "" - if referrer != nil { - baseURL = referrer.URL.String() - } - - // resolve URL into a full absolute URL (non-relative) - u, err := resolveURL(baseURL, rawurl) - if err != nil { - return nil, err - } - - // create struct and return - return &Page{URL: u, Skipped: false, Broken: false}, nil -} diff --git a/src/docrawler/page_test.go b/src/docrawler/page_test.go deleted file mode 100644 index 217f290..0000000 --- a/src/docrawler/page_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "testing" -) - -// TestBadNewPage tests NewPage to see if we are getting expected failures -func TestBadNewPage(t *testing.T) { - _, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/") - if err == nil { - t.Error("tried creating a NewPage with an invalid URL but didn't get an error") - } -} diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 1c735d6..4eb31b3 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -11,7 +11,7 @@ var reTitle = regexp.MustCompile(`(?i)<\s*title\s*>([^<]*)<\s*\/\s*title`) // parseLinks takes a string and attempts to parse any html title and all links out of it, // and returns a slice of the captures found -func parseLinks(s string) (string, []string, error) { +func parseLinks(s string) (string, []string) { // find the title, or default to empty title := "" titleMatches := reTitle.FindStringSubmatch(s) @@ -25,5 +25,5 @@ func parseLinks(s string) (string, []string, error) { for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } - return title, results, nil + return title, results } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index a04b869..ccc08a3 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -16,10 +16,7 @@ func TestSimpleParse(t *testing.T) {