From be11b13443ccb8f5b7c5f8f4b8b1d8905195e0f0 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Sun, 15 Feb 2015 18:57:34 -0800 Subject: [PATCH] json sitemap output now functional * the json marshaller cannot deal with circular data structures, and no automatic .String() conversion * so we copy our crawl data slice into another slice which will be marshaled into json more easily * made the page slice we use during crawling actually circular (as it should be) rather than putting duplicates in for each page discovered * test for conversion to output data structure, and hacky cheater test for json output * removed Base *Page from our Page struct as it's only needed temporarily --- src/docrawler/docrawler.go | 28 ++++++++++------ src/docrawler/docrawler_test.go | 50 +++++++++++++++++++++++++++-- src/docrawler/output.go | 57 +++++++++++++++++++++++++++++++++ src/docrawler/page.go | 1 - 4 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 src/docrawler/output.go diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index f66e9e0..9f4f19c 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,7 +1,5 @@ package main -import () - // docrawl begins crawling the site at "url" func docrawl(url string) []*Page { // our result structure @@ -35,18 +33,30 @@ func docrawl(url string) []*Page { } // parse links - _, links, err := parseLinks(text) + title, links, err := parseLinks(text) if err != nil { continue // TODO handle } + dest.Title = title for _, l := range links { - p, _ := NewPage(dest, l) // TODO check error - dest.Children = append(dest.Children, p) + p, err := NewPage(dest, l) + if err != nil { + continue // TODO check error + } - // see if we should queue this new page for crawling - _, haveCrawled := crawled[p.URL.String()] - _, haveQueued := queued[p.URL.String()] - if !haveCrawled && !haveQueued { + // see if we already know about this page + pCrawled, haveCrawled := crawled[p.URL.String()] + pQueued, haveQueued := queued[p.URL.String()] + if haveCrawled { + // add the previously crawled page to children + dest.Children = append(dest.Children, pCrawled) + } else if haveQueued { + // add the previously queued page to children + dest.Children = append(dest.Children, pQueued) + } else { + // add this new page (never seen) to children + dest.Children = append(dest.Children, p) + // .. and queue it up for crawling queued[p.URL.String()] = p } } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index a9c565e..ff387f9 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -18,13 +18,13 @@ func TestMain(m *testing.M) { // start our simple web server l, err := net.Listen("tcp", ":"+strconv.Itoa(port)) if err != nil { - panic("unable to open port for http server") // TODO remove panic? + panic("unable to open port for http server") } http.Handle("/", http.FileServer(http.Dir("./testsite"))) go func() { err = http.Serve(l, nil) if err != nil { - panic("unable to start http server") // TODO remove panic? + panic("unable to start http server") } }() @@ -132,3 +132,49 @@ func TestHeaderFetching(t *testing.T) { t.Error("problem fetching filetype") } } + +// TestJsonOutput gets a sitemap and then converts it to json +func TestJsonOutput(t *testing.T) { + pages := docrawl(baseURL) + l := sitemapToLocations(pages) + if len(l) != 2 { + t.Error("sitemapToLocations has the wrong number of locations") + } + if l[0].URL != baseURL { + t.Error("got an incorrect location") + } + if len(l[0].Links) != 1 { + t.Error("location 0 has wrong number of links") + } + if len(l[0].Assets) != 2 { + t.Error("location 0 has wrong number of assets") + } + if len(l[0].Broken) != 0 { + t.Error("location 0 has wrong number of broken urls") + } + if l[1].URL != baseURL+"about.html" { + t.Error("got an incorrect location") + } + if len(l[1].Links) != 1 { + t.Error("location 1 has wrong number of links") + } + if len(l[1].Assets) != 2 { + t.Error("location 1 has wrong number of assets") + } + if len(l[1].Broken) != 1 { + t.Error("location 1 has wrong number of broken urls") + } + + // this is a cheater test because the output is from a run of the code being + // test itself. but, it's been examined and I think it's right. and, there's not + // many other ways to test this without some significant pain + cheaterTest := `[{"URL":"http://localhost:8765/","Title":"Home","Links":["http://localhost:8765/about.html"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":null},{"URL":"http://localhost:8765/about.html","Title":"About Test","Links":["http://localhost:8765/"],"Assets":["http://localhost:8765/assets/image.png","http://localhost:8765/scripts/blah.js"],"Broken":["http://doesntexist23492387492837492374982734.com/"]}]` + j, err := locationsToJSON(l) + if err != nil { + t.Error("locationsToJSON failed") + } + if j != cheaterTest { + t.Error("locationsToJSON text doesn't match") + } + t.Log(j) +} diff --git a/src/docrawler/output.go b/src/docrawler/output.go new file mode 100644 index 0000000..50a2880 --- /dev/null +++ b/src/docrawler/output.go @@ -0,0 +1,57 @@ +package main + +import ( + "encoding/json" +) + +// Location is a struct which defines a single URL, which URLs (links and assets) it contains, etc. +type Location struct { + URL string + Title string + Links []string + Assets []string + Broken []string +} + +func sitemapToLocations(pages []*Page) []*Location { + // build a map of our pages + pageMap := make(map[string]*Page) + for _, p := range pages { + pageMap[p.URL.String()] = p + } + + // build a slice of locations (one per page) + var locations []*Location + for _, p := range pages { + if p.MediaType == "text/html" { + // create a location for this page + l := &Location{URL: p.URL.String(), Title: p.Title} + + // add its children + for _, c := range p.Children { + // look up this child's media type from the root list of pages + //mediaType := pageMap[c.URL.String()].MediaType + if c.MediaType == "text/html" { + l.Links = append(l.Links, c.URL.String()) + } else if c.MediaType == "" { + l.Broken = append(l.Broken, c.URL.String()) + } else { + l.Assets = append(l.Assets, c.URL.String()) + } + } + + // and add this location to our slice + locations = append(locations, l) + } + } + + return locations +} + +func locationsToJSON(locations []*Location) (string, error) { + b, err := json.Marshal(locations) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/src/docrawler/page.go b/src/docrawler/page.go index 37a7f7c..4e52304 100644 --- a/src/docrawler/page.go +++ b/src/docrawler/page.go @@ -7,7 +7,6 @@ import ( // Page is a struct which defines a single page, which URLs (links and assets) it contains, etc. type Page struct { URL *url.URL - Base *Page Title string MediaType string Children []*Page