swaits/docrawler
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

renamed Page -> httpItem, and changed its members to unexported as well

Browse Source 
* tests all passing
This commit is contained in:
swaits committed 2015-02-16 18:14:12 -08:00
1 parent 66f78b562f
commit 58614802b6
7 files changed
+97 -93
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/docrawler/docrawler.go
+24 -20
View File
@@ -5,23 +5,27 @@ import (
"os"
)
type itemSlice []*httpItem
type itemMap map[string]*httpItem
// docrawl begins crawling the site at "url"
func docrawl(url string) []*Page {
func docrawl(url string) itemSlice {
// our result structure
var sitemap []*Page
var sitemap itemSlice
// what we've already parsed, what we still need to parse
crawled := make(map[string]*Page)
queued := make(map[string]*Page)
crawled := make(itemMap)
queued := make(itemMap)
// create first page and add to the queue
homepage, _ := NewPage(nil, url) // TODO check error
queued[homepage.URL.String()] = homepage
homepage, _ := newHTTPItem(nil, url) // TODO check error
queued[homepage.url.String()] = homepage
// main loop
for len(queued) > 0 {
// pop a page off of queued
var dest *Page
var dest *httpItem
var destURL string
for destURL, dest = range queued {
break
@@ -29,52 +33,52 @@ func docrawl(url string) []*Page {
delete(queued, destURL)
// mark this page as complete
crawled[dest.URL.String()] = dest
crawled[dest.url.String()] = dest
// add page to list
sitemap = append(sitemap, dest)
// make sure this URL has the same hostname as our first page
if dest.URL.Host != homepage.URL.Host {
if dest.url.Host != homepage.url.Host {
// skip URLs associated with other Hosts
dest.Skipped = true
dest.skipped = true
continue
}
// fetch page
text, err := fetchPage(dest)
if err != nil {
dest.Broken = true
dest.broken = true
continue
}
// parse links
title, links, err := parseLinks(text)
if err != nil {
dest.Broken = true
dest.broken = true
continue
}
dest.Title = title
dest.title = title
for _, l := range links {
p, err := NewPage(dest, l)
p, err := newHTTPItem(dest, l)
if err != nil {
continue // TODO check error
}
// see if we already know about this page
pCrawled, haveCrawled := crawled[p.URL.String()]
pQueued, haveQueued := queued[p.URL.String()]
pCrawled, haveCrawled := crawled[p.url.String()]
pQueued, haveQueued := queued[p.url.String()]
if haveCrawled {
// add the previously crawled page to children
dest.Children = append(dest.Children, pCrawled)
dest.children = append(dest.children, pCrawled)
} else if haveQueued {
// add the previously queued page to children
dest.Children = append(dest.Children, pQueued)
dest.children = append(dest.children, pQueued)
} else {
// add this new page (never seen) to children
dest.Children = append(dest.Children, p)
dest.children = append(dest.children, p)
// .. and queue it up for crawling
queued[p.URL.String()] = p
queued[p.url.String()] = p
}
}
}
src/docrawler/docrawler_test.go
+33 -33
View File
@@ -58,9 +58,9 @@ func TestBadRequest(t *testing.T) {
// TestSimpleFetch makes sure we can fetch a file and we get exactly what we expect
func TestSimpleFetch(t *testing.T) {
page, err := NewPage(nil, baseURL+"fetch_test.html")
page, err := newHTTPItem(nil, baseURL+"fetch_test.html")
if err != nil {
t.Error("problem creating New Page struct")
t.Error("problem creating New httpItem struct")
}
body, err := fetchPage(page)
if err != nil {
@@ -82,19 +82,19 @@ func TestSimpleMap(t *testing.T) {
// cycle through every page, counting stuff, finding specific pages
// so that we can then verify certain aspects of each page
skipped, children := 0, 0
var homepage, aboutpage, skippage *Page
var homepage, aboutpage, skippage *httpItem
for _, p := range pages {
if p.Title == "Home" {
if p.title == "Home" {
homepage = p
} else if p.Title == "About Test" {
} else if p.title == "About Test" {
aboutpage = p
} else if p.URL.String() == "http://doesntexist23492387492837492374982734.com/" {
} else if p.url.String() == "http://doesntexist23492387492837492374982734.com/" {
skippage = p
}
if p.Skipped {
if p.skipped {
skipped++
}
children += len(p.Children)
children += len(p.children)
}
// check page counts
@@ -112,44 +112,44 @@ func TestSimpleMap(t *testing.T) {
}
// check first page URL
if homepage.URL.String() != baseURL {
if homepage.url.String() != baseURL {
t.Error("page URL is invalid")
}
// specific page number of children
if len(homepage.Children) != 4 {
t.Logf("got %v, wanted %v\n", len(homepage.Children), 4)
if len(homepage.children) != 4 {
t.Logf("got %v, wanted %v\n", len(homepage.children), 4)
t.Fatal("got wrong number of links")
}
if len(aboutpage.Children) != 4 {
t.Logf("got %v, wanted %v\n", len(aboutpage.Children), 4)
if len(aboutpage.children) != 4 {
t.Logf("got %v, wanted %v\n", len(aboutpage.children), 4)
t.Fatal("got wrong number of links")
}
// make sure remote pages were skipped
if homepage.Skipped != false {
t.Logf("found a page that should NOT have been skipped %q", homepage.Children[0].URL.String())
if homepage.skipped != false {
t.Logf("found a page that should NOT have been skipped %q", homepage.children[0].url.String())
t.Error("child was skipped")
}
if skippage.Skipped != true {
t.Logf("found a page that should have been skipped %q", pages[3].Children[2].URL.String())
if skippage.skipped != true {
t.Logf("found a page that should have been skipped %q", pages[3].children[2].url.String())
t.Error("child wasn't skipped")
}
// verify page content
// TODO find a better way to do this
//if homepage.Children[0].URL.String() != baseURL+"about.html" {
//t.Logf(" Got: %q\n", homepage.Children[0].URL.String())
//if homepage.children[0].url.String() != baseURL+"about.html" {
//t.Logf(" Got: %q\n", homepage.children[0].url.String())
//t.Logf("Wanted: %q\n", baseURL+"about.html")
//t.Error("link name is incorrect")
//}
//if homepage.Children[1].URL.String() != baseURL+"assets/image.png" {
//t.Logf(" Got: %q\n", homepage.Children[1].URL.String())
//if homepage.children[1].url.String() != baseURL+"assets/image.png" {
//t.Logf(" Got: %q\n", homepage.children[1].url.String())
//t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
//t.Error("link name is incorrect")
//}
//if homepage.Children[3].URL.String() != baseURL+"scripts/blah.js" {
//t.Logf(" Got: %q\n", homepage.Children[3].URL.String())
//if homepage.children[3].url.String() != baseURL+"scripts/blah.js" {
//t.Logf(" Got: %q\n", homepage.children[3].url.String())
//t.Logf("Wanted: %q\n", baseURL+"scripts/blah.js")
//t.Error("link name is incorrect")
//}
@@ -157,28 +157,28 @@ func TestSimpleMap(t *testing.T) {
// TestHeaderFetching tests fetchFiletype() to see if we are getting expected results
func TestHeaderFetching(t *testing.T) {
basepage, err := NewPage(nil, baseURL)
basepage, err := newHTTPItem(nil, baseURL)
if err != nil {
t.Error("problem creating New Page struct")
t.Error("problem creating New httpItem struct")
}
if err := fetchFiletype(basepage); err != nil || basepage.MediaType != "text/html" {
if err := fetchFiletype(basepage); err != nil || basepage.mediaType != "text/html" {
t.Error("problem fetching filetype")
}
page, err := NewPage(basepage, "about.html")
page, err := newHTTPItem(basepage, "about.html")
if err != nil {
t.Error("problem creating New Page struct")
t.Error("problem creating New httpItem struct")
}
if err := fetchFiletype(page); err != nil || page.MediaType != "text/html" {
if err := fetchFiletype(page); err != nil || page.mediaType != "text/html" {
t.Error("problem fetching filetype")
}
page, err = NewPage(basepage, "assets/image.png")
page, err = newHTTPItem(basepage, "assets/image.png")
if err != nil {
t.Error("problem creating New Page struct")
t.Error("problem creating New httpItem struct")
}
if err := fetchFiletype(page); err != nil || page.MediaType != "image/png" {
t.Logf("got %q, wanted %q", page.MediaType, "image/png")
if err := fetchFiletype(page); err != nil || page.mediaType != "image/png" {
t.Logf("got %q, wanted %q", page.mediaType, "image/png")
t.Error("problem fetching filetype")
}
}
src/docrawler/fetch.go
+8 -8
View File
@@ -13,9 +13,9 @@ var (
)
// fetchFiletype performs an http HEAD to get the media type, and sets it
// directly in Page.MediaType
func fetchFiletype(page *Page) error {
resp, err := http.Head(page.URL.String())
// directly in httpItem.mediaType
func fetchFiletype(page *httpItem) error {
resp, err := http.Head(page.url.String())
if err != nil {
return err
}
@@ -39,12 +39,12 @@ func fetchFiletype(page *Page) error {
}
// set and return
page.MediaType = mediatype
page.mediaType = mediatype
return nil
}
// fetchPage takes a Page, GETs it, and returns the body as a string
func fetchPage(page *Page) (string, error) {
// fetchPage takes a httpItem, GETs it, and returns the body as a string
func fetchPage(page *httpItem) (string, error) {
// figure out the file type
err := fetchFiletype(page)
if err != nil {
@@ -52,12 +52,12 @@ func fetchPage(page *Page) (string, error) {
}
// we only want to fetch html
if page.MediaType != "text/html" {
if page.mediaType != "text/html" {
return "", nil // but this isn't an error!
}
// GET the url
resp, err := http.Get(page.URL.String())
resp, err := http.Get(page.url.String())
if err != nil {
return "", err
}
src/docrawler/fetch_test.go
+2 -2
View File
@@ -6,7 +6,7 @@ import (
// TestFailedHeaderFetching tests fetchFiletype() to see if we are getting expected failures
func TestFailedHeaderFetching(t *testing.T) {
page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
if err != nil {
t.Error("problem creating New Page struct")
}
@@ -17,7 +17,7 @@ func TestFailedHeaderFetching(t *testing.T) {
// TestFailedFetching tests fetchFiletype() to see if we are getting expected failures
func TestFailedFetching(t *testing.T) {
page, err := NewPage(nil, "http://doesntexist23492387492837492374982734.com/")
page, err := newHTTPItem(nil, "http://doesntexist23492387492837492374982734.com/")
if err != nil {
t.Error("problem creating New Page struct")
}
src/docrawler/output.go
+14 -14
View File
@@ -15,32 +15,32 @@ type Location struct {
Remote []string
}
func sitemapToLocations(pages []*Page) []*Location {
func sitemapToLocations(pages []*httpItem) []*Location {
// build a map of our pages
pageMap := make(map[string]*Page)
pageMap := make(map[string]*httpItem)
for _, p := range pages {
pageMap[p.URL.String()] = p
pageMap[p.url.String()] = p
}
// build a slice of locations (one per page)
var locations []*Location
for _, p := range pages {
if p.MediaType == "text/html" {
if p.mediaType == "text/html" {
// create a location for this page
l := &Location{URL: p.URL.String(), Title: p.Title}
l := &Location{URL: p.url.String(), Title: p.title}
// add its children
for _, c := range p.Children {
for _, c := range p.children {
// look up this child's media type from the root list of pages
//mediaType := pageMap[c.URL.String()].MediaType
if c.Skipped {
l.Remote = append(l.Remote, c.URL.String())
} else if c.MediaType == "text/html" {
l.Links = append(l.Links, c.URL.String())
} else if c.Broken {
l.Broken = append(l.Broken, c.URL.String())
//mediaType := pageMap[c.url.String()].mediaType
if c.skipped {
l.Remote = append(l.Remote, c.url.String())
} else if c.mediaType == "text/html" {
l.Links = append(l.Links, c.url.String())
} else if c.broken {
l.Broken = append(l.Broken, c.url.String())
} else {
l.Assets = append(l.Assets, c.URL.String())
l.Assets = append(l.Assets, c.url.String())
}
}
src/docrawler/page.go
+12 -12
View File
@@ -4,22 +4,22 @@ import (
"net/url"
)
// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
type Page struct {
URL *url.URL
Title string
MediaType string
Skipped bool
Broken bool
Children []*Page
// httpItem is a struct which defines a single page, which URLs (links and assets) it contains, etc.
type httpItem struct {
url *url.URL
title string
mediaType string
skipped bool
broken bool
children []*httpItem
}
// NewPage takes a referring Page + a URL and returns a new &Page{}
func NewPage(referrer *Page, rawurl string) (*Page, error) {
// newHTTPItem takes a referring httpItem + a URL and returns a new &httpItem{}
func newHTTPItem(referrer *httpItem, rawurl string) (*httpItem, error) {
// determine the base URL so we can resulve this into a full URL
baseURL := ""
if referrer != nil {
baseURL = referrer.URL.String()
baseURL = referrer.url.String()
}
// resolve URL into a full absolute URL (non-relative)
@@ -29,5 +29,5 @@ func NewPage(referrer *Page, rawurl string) (*Page, error) {
}
// create struct and return
return &Page{URL: u, Skipped: false, Broken: false}, nil
return &httpItem{url: u, skipped: false, broken: false}, nil
}
src/docrawler/page_test.go
+4 -4
View File
@@ -4,10 +4,10 @@ import (
"testing"
)
// TestBadNewPage tests NewPage to see if we are getting expected failures
func TestBadNewPage(t *testing.T) {
_, err := NewPage(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
// TestBadnewHTTPItem tests newHTTPItem to see if we are getting expected failures
func TestBadnewHTTPItem(t *testing.T) {
_, err := newHTTPItem(nil, "h%20t\tp://doesntexist23492387492837492374982734.com/")
if err == nil {
t.Error("tried creating a NewPage with an invalid URL but didn't get an error")
t.Error("tried creating a newHTTPItem with an invalid URL but didn't get an error")
}
}