From 6c9434e39a97768c15dc5bdc2bbddc1a751bc14b Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Tue, 17 Feb 2015 09:53:28 -0800 Subject: [PATCH 1/3] flow: Created branch 'feature/cleanup_and_refactoring'. From 3e35fdb48802f82d022b819725fe2d832d384634 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Tue, 17 Feb 2015 10:36:48 -0800 Subject: [PATCH 2/3] some cleanup and refactoring * moved crawlItem into fetch.go * renamed fetchPage fetchItem * doCrawl now takes number of workers as a parameter * command line flag parsing added --- src/docrawler/docrawler.go | 78 +++++++++++---------------------- src/docrawler/docrawler_test.go | 6 +-- src/docrawler/fetch.go | 32 +++++++++++++- src/docrawler/fetch_test.go | 2 +- 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index 26a84f8..780f84b 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -1,6 +1,7 @@ package main import ( + "flag" "fmt" "log" "os" @@ -8,7 +9,7 @@ import ( ) // doCrawl begins crawling the site at "homeurl" -func doCrawl(homeurl string) itemSlice { +func doCrawl(homeurl string, nWorkers int) itemSlice { // set of what we have already crawled, our results crawled := make(itemMap) @@ -22,8 +23,8 @@ func doCrawl(homeurl string) itemSlice { txchan := make(chan *httpItem) // spin up our crawler workers - for i := 0; i < 10; i++ { - go crawlListner(txchan, rxchan) + for i := 0; i < nWorkers; i++ { + go crawlWorker(txchan, rxchan) } // start a ticker which we'll use to output status and check for completion @@ -109,65 +110,36 @@ func doCrawl(homeurl string) itemSlice { } } -// crawlListner is a for+select wrapper around crawlItem that listens -// for new jobs and sends them off to crawlItem -func crawlListner(txchan <-chan *httpItem, rxchan chan<- *httpItem) { - for { - select { - case newJob, ok := <-txchan: - if !ok { - // channel closed, finish - return - } - // perform the crawl - rxchan <- crawlItem(newJob) - } +// crawlWorker is a goroutine'ized wrapper around crawlItem that listens +// for new jobs and sends them off to crawlItem, returning the results in rxchan +func crawlWorker(txchan <-chan *httpItem, rxchan chan<- *httpItem) { + // read off the incoming item channel forever + for newJob := range txchan { + // perform the crawl + crawlItem(newJob) + // return the result + rxchan <- newJob } } -// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, -// and filling out its structure as much as possible -func crawlItem(item *httpItem) *httpItem { - // make sure this item is the same domain (i.e. URL "host part") as its referrer - if item.refurl != nil && item.url.Host != item.refurl.Host { - // skip URLs associated with other Hosts - item.linkType = tRemote - return item - } - - // fetch page - text, err := fetchPage(item) - if err != nil { - return item - } - - // parse links - title, links := parseLinks(text) - item.title = title - - // walk links and add them as children to the current item - for _, l := range links { - newItem, err := newHTTPItem(item, l) - if err != nil { - continue // TODO bad item - } - item.children = append(item.children, newItem) - } - - // send back our item struct now that it's all filled out - return item -} - +// main is our program's entry point func main() { + // parse our flags + nWorkers := *(flag.Uint("num", 100, "number of workers")) // see if we've got no arguments - if len(os.Args) == 1 { + flag.Parse() + if flag.NArg() < 1 { + fmt.Printf("\nerror: Please specify at least one URL to crawl.\n") fmt.Printf("\nD.O. Crawler 1.0 Copyright (c) 2015 Stephen Waits 2015-02-17\n\n") - fmt.Printf("usage: %v \n\n", os.Args[0]) + fmt.Printf("usage: %v [-num=100] \n\n", os.Args[0]) + fmt.Printf(" -num=100: number of workers\n") + fmt.Printf(" URLs: URLs to crawl\n\n") os.Exit(1) } + // crawl each URL on the command line - for _, u := range os.Args[1:] { - pages := doCrawl(u) + for _, u := range flag.Args() { + pages := doCrawl(u, int(nWorkers)) if pages == nil { log.Fatalf("unable to crawl %q, may be an invalid URL\n", u) } diff --git a/src/docrawler/docrawler_test.go b/src/docrawler/docrawler_test.go index 60bd8cc..5c11cc9 100644 --- a/src/docrawler/docrawler_test.go +++ b/src/docrawler/docrawler_test.go @@ -62,7 +62,7 @@ func TestSimpleFetch(t *testing.T) { if err != nil { t.Error("problem creating New httpItem struct") } - body, err := fetchPage(page) + body, err := fetchItem(page) if err != nil { t.Fatal(err) } @@ -76,7 +76,7 @@ func TestSimpleFetch(t *testing.T) { // TestSimpleMap figures out the site map for the site in baseURL func TestSimpleMap(t *testing.T) { - pages := doCrawl(baseURL) + pages := doCrawl(baseURL, 10) // because our crawl is non-deterministic, we have to do a complete // cycle through every page, counting stuff, finding specific pages @@ -185,7 +185,7 @@ func TestHeaderFetching(t *testing.T) { // TestJsonOutput gets a sitemap and then converts it to json func TestJsonOutput(t *testing.T) { - pages := doCrawl(baseURL) + pages := doCrawl(baseURL, 10) l := sitemapToLocations(pages) if len(l) != 2 { t.Error("sitemapToLocations has the wrong number of locations") diff --git a/src/docrawler/fetch.go b/src/docrawler/fetch.go index bab34f0..a24c954 100644 --- a/src/docrawler/fetch.go +++ b/src/docrawler/fetch.go @@ -54,7 +54,7 @@ func fetchFiletype(item *httpItem) error { } // fetchPage takes a httpItem, GETs it, and returns the body as a string -func fetchPage(item *httpItem) (string, error) { +func fetchItem(item *httpItem) (string, error) { // figure out the file type err := fetchFiletype(item) if err != nil { @@ -92,3 +92,33 @@ func fetchPage(item *httpItem) (string, error) { resp.Body.Close() return string(body), nil } + +// crawlItem crawls a single httpItem, fetching the header, hte page, parsing it, +// and filling out its structure as much as possible +func crawlItem(item *httpItem) { + // make sure this item is the same domain (i.e. URL "host part") as its referrer + if item.refurl != nil && item.url.Host != item.refurl.Host { + // skip URLs associated with other Hosts + item.linkType = tRemote + return + } + + // fetch page + text, err := fetchItem(item) + if err != nil { + return + } + + // parse links + title, links := parseLinks(text) + item.title = title + + // walk links and add them as children to the current item + for _, l := range links { + newItem, err := newHTTPItem(item, l) + if err != nil { + continue // TODO bad item + } + item.children = append(item.children, newItem) + } +} diff --git a/src/docrawler/fetch_test.go b/src/docrawler/fetch_test.go index 0cf42d1..c666a87 100644 --- a/src/docrawler/fetch_test.go +++ b/src/docrawler/fetch_test.go @@ -21,7 +21,7 @@ func TestFailedFetching(t *testing.T) { if err != nil { t.Error("problem creating New Page struct") } - if _, err := fetchPage(page); err == nil { + if _, err := fetchItem(page); err == nil { t.Error("tired fetching bogus page but didn't get nil back from fetchPage") } } From 154ef4321e09ca73b29475693286d3092590ecf6 Mon Sep 17 00:00:00 2001 From: Stephen Waits Date: Tue, 17 Feb 2015 10:38:04 -0800 Subject: [PATCH 3/3] flow: Closed 'cleanup_and_refactoring'.