diff --git a/README.md b/README.md index f64d7f1..66aea59 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,51 @@ It crawls a single host (i.e. anything.com) and outputs a site map in JSON forma For each page crawled, it distinguishes between links to other pages, links to assets, broken links, and remote links (i.e. someotherhost.com). +### An Example + + ❯ bin/docrawler https://goregex.com/ + + D.O. Crawler 1.0 Copyright (c) 2015 Stephen Waits 2015-02-17 + + [ + { + "URL": "https://goregex.com/", + "Title": "GoRegEx.com | Go Regular Expression Tester", + "Links": [ + "https://goregex.com/" + ], + "Assets": [ + "https://goregex.com/assets/css/bootstrap-responsive.min.css", + "https://goregex.com/assets/css/bootstrap.min.css", + "https://goregex.com/assets/css/goregex.css", + "https://goregex.com/assets/img/apple-touch-icon-114x114.png", + "https://goregex.com/assets/img/apple-touch-icon-120x120.png", + "https://goregex.com/assets/img/apple-touch-icon-144x144.png", + "https://goregex.com/assets/img/apple-touch-icon-152x152.png", + "https://goregex.com/assets/img/apple-touch-icon-180x180.png", + "https://goregex.com/assets/img/apple-touch-icon-57x57.png", + "https://goregex.com/assets/img/apple-touch-icon-60x60.png", + "https://goregex.com/assets/img/apple-touch-icon-72x72.png", + "https://goregex.com/assets/img/apple-touch-icon-76x76.png", + "https://goregex.com/assets/img/favicon-160x160.png", + "https://goregex.com/assets/img/favicon-16x16.png", + "https://goregex.com/assets/img/favicon-192x192.png", + "https://goregex.com/assets/img/favicon-32x32.png", + "https://goregex.com/assets/img/favicon-96x96.png", + "https://goregex.com/assets/img/favicon.ico", + "https://goregex.com/assets/js/goregex.js" + ], + "Broken": null, + "Remote": [ + "http://code.google.com/p/re2/wiki/Syntax", + "http://golang.org/", + "http://golang.org/pkg/regexp/", + "https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js", + "https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js" + ] + } + ] + ## Design ## * The main crawler loop is in `docrawler.go:docrawl()`. @@ -21,6 +66,8 @@ After cloning this repository, run: make vendor && make +This will `fmt`, `lint`, `vet`, and `build` the source into an executable at `bin/docrawler`. + In order to run the tests: make test @@ -29,10 +76,25 @@ If you'd like to automatically run tests any time a file is changed: make autotest +To see stats about the code, install cloc (`brew install cloc` on OS X) and then: + + make stats + +To look at the test coverage reports in your browser: + + make cover + To deploy, just copy the executable `bin/docrawler` to a destination of your choosing. *Developed on OS X. Also tested on Windows 7 under MinGW+bash.* +### Why a Makefile? Aren't those old??? + +* It's clean. +* It's portable. +* It lets me package the app in a way that's trivially easy for anyone to build. +* It makes it so I can easily vendor third-party packages outside of the main `./src` tree. + ## Who do I talk to? ## Written by Stephen Waits. Please contact me at with any questions. diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go index ff0b233..bb93ef7 100644 --- a/src/docrawler/docrawler.go +++ b/src/docrawler/docrawler.go @@ -124,13 +124,13 @@ func crawlWorker(txchan <-chan *httpItem, rxchan chan<- *httpItem) { // main is our program's entry point func main() { + fmt.Printf("\nD.O. Crawler 1.0 Copyright (c) 2015 Stephen Waits 2015-02-17\n\n") // parse our flags nWorkers := *(flag.Uint("num", 100, "number of workers")) // see if we've got no arguments flag.Parse() if flag.NArg() < 1 { - fmt.Printf("\nerror: Please specify at least one URL to crawl.\n") - fmt.Printf("\nD.O. Crawler 1.0 Copyright (c) 2015 Stephen Waits 2015-02-17\n\n") + fmt.Printf("error: Please specify at least one URL to crawl.\n\n") fmt.Printf("usage: %v [-num=100] \n\n", os.Args[0]) fmt.Printf(" -num=100: number of workers\n") fmt.Printf(" URLs: URLs to crawl\n\n")