From b0e64c7973fad43a336b5bd15a301189fad452bb Mon Sep 17 00:00:00 2001
From: Stephen Waits <steve@waits.net>
Date: Sat, 14 Feb 2015 21:00:18 -0800
Subject: [PATCH] initial commit, empty go project, plus simple design doc

---
 Makefile                   | 55 ++++++++++++++++++++++++++++++++++++++
 docs/Design.md             | 45 +++++++++++++++++++++++++++++++
 src/docrawler/docrawler.go |  6 +++++
 3 files changed, 106 insertions(+)
 create mode 100644 Makefile
 create mode 100644 docs/Design.md
 create mode 100644 src/docrawler/docrawler.go

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..2dc1afd
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,55 @@
+# Golang project Makefile
+#
+# derived from: http://zduck.com/2014/go-project-structure-and-dependencies/
+
+# build targets (which aren't files)
+.PHONY: build fmt lint vet test autotest run clean vendor
+
+# configuration
+APPNAME := docrawler
+VENDOR := .vendor
+PACKAGES :=                     \
+	golang.org/x/tools/cmd/vet    \
+	github.com/golang/lint/golint
+
+# override GOPATH to be our vendor directory
+export GOPATH := ${PWD}/${VENDOR}
+export PATH := ${PWD}/${VENDOR}/bin:${PATH}
+
+default: build
+
+build: fmt lint vet
+	go build -v -o ./bin/${APPNAME} ./src/${APPNAME}
+
+fmt:
+	go fmt ./src/...
+
+lint:
+	golint ./src
+
+vet:
+	go vet ./src/...
+
+test:
+	go test ./src/...
+
+autotest:
+	fswatch -or ./src | xargs -n1 -I{} go test ./src/...
+
+run: build
+	./bin/${APPNAME}
+
+clean:
+	rm -f ./bin/${APPNAME}
+
+# vendor
+# * wipes our vendor directory completely
+# * fetches all packages in ${PACKAGES}
+# * removes source control dirs so packages are vendorable (commitable)
+vendor:
+	rm -dRf ./${VENDOR}/src                          && \
+	go get -u ${PACKAGES}                            && \
+	rm -rf `find ./${VENDOR}/src -type d -name .git` && \
+	rm -rf `find ./${VENDOR}/src -type d -name .hg`  && \
+	rm -rf `find ./${VENDOR}/src -type d -name .bzr` && \
+	rm -rf `find ./${VENDOR}/src -type d -name .svn`
diff --git a/docs/Design.md b/docs/Design.md
new file mode 100644
index 0000000..4b2a587
--- /dev/null
+++ b/docs/Design.md
@@ -0,0 +1,45 @@
+# Crawler Requirements
+
+## Minimum Features
+
+This is the assignment (directly) from DO.
+
+* We do ask that you put a heavy emphasis on testing in the assignment.
+* written in a modern language
+* limited to one domain - so when crawling digitalocean.com it would crawl all pages within the digitalocean.com domain, but not follow the links to our Facebook or Instagram accounts or subdomains like cloud.digitalocean.com. 
+* It should be Given a URL
+* it should output a site map
+* it should show which static assets each page depends on, and the links between pages.
+* Choose the most appropriate data structure to store & display this site map.
+* Build this as you would build something for production - it's fine if you don't finish everything, but focus on code quality and write tests.
+* We're interested in how you code and how you test your code.
+
+## Desired
+
+These are features I'd like to include.
+
+* efficient, optimal
+* handles base URL properly (for root & relative links)
+* documented
+* uses standard Go practices
+* uses Go stdlib (only!)
+* thoroughly tested
+* configurable on command line (defaults are for company specs)
+* variable output formats: json, dot
+* adheres to URL RFC (as far as case sensitivity, acceptable character sets, etc.)
+* go gettable
+* go fmt'ed, vet'ed, lint'ed
+* polite - robots.txt support
+* robust - detect infinite loops
+* throttling
+* use Go context pattern
+* supports http & https
+
+## Wishlist
+
+These are features that sound cool but are probably out of scope.
+
+* distributed
+* sitemap support https://en.wikipedia.org/wiki/Sitemaps
+* noindex tag
+* noidext http header
diff --git a/src/docrawler/docrawler.go b/src/docrawler/docrawler.go
new file mode 100644
index 0000000..c8f4ae0
--- /dev/null
+++ b/src/docrawler/docrawler.go
@@ -0,0 +1,6 @@
+package main
+
+import ()
+
+func main() {
+}