swaits/docrawler
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

minimally functional crawler - crawls single page, finds links, stores in data structure

Browse Source 
* refined Page struct. now need to refactor everything else to use this
* wrote docrawl() function. NOT intended to be anything like the final design. just sketching it out to get a better idea of needs, design, etc.
* replaced fmt.Print* with t.Log* in tests (oops!)
* added a header fetcher function to get the file type (since we only want to parse html (with tests)
* added small test for the URL resolver where the base URL is empty
* added placeholder files in testsite/ for about.html and image.png
This commit is contained in:
swaits committed 2015-02-15 12:12:30 -08:00
1 parent 7f6978ad27
commit 0456723f0b
6 files changed
+123 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/docrawler/docrawler.go
+56 -8
View File
@@ -1,18 +1,66 @@
package main
import ()
import (
"net/url"
)
// Page is a struct which defines a single page, which URLs (links and assets) it contains
// Page is a struct which defines a single page, which URLs (links and assets) it contains, etc.
type Page struct {
URL string
Assets []string
Links []string
URL *url.URL
Base *Page
Title string
MediaType string
Children []*Page
}
// docrawl begins crawling the site at "url"
func docrawl(url string) []Page {
var result []Page
return result
func docrawl(url string) []*Page {
// our result structure
var sitemap []*Page
// what we've already parsed, what we still need to parse
parsed := make(map[string]*Page)
var queued []*Page
// create first page and add to the queue
u, _ := resolveURL("", url) // TODO check error
queued = append(queued, &Page{URL: u})
// main loop
for len(queued) > 0 {
// pop a page off of queued
var dest *Page
dest, queued = queued[len(queued)-1], queued[:len(queued)-1]
// skip loop if we already did this URL
if _, ok := parsed[dest.URL.String()]; ok {
continue // TODO handle
}
// mark this page as complete
parsed[dest.URL.String()] = dest
// fetch page
text, err := fetch(dest.URL.String())
if err != nil {
continue // TODO handle
}
// parse it
_, links, err := parse(text)
if err != nil {
continue // TODO handle
}
for _, l := range links {
u, _ := resolveURL(dest.URL.String(), l) // TODO check error
dest.Children = append(dest.Children, &Page{URL: u, Base: dest})
}
// add page to list
sitemap = append(sitemap, dest)
}
return sitemap
}
func main() {
src/docrawler/docrawler_test.go
+31 -14
View File
@@ -1,7 +1,6 @@
package main
import (
"fmt"
"net"
"net/http"
"os"
@@ -43,7 +42,7 @@ func TestMain(m *testing.M) {
func TestServerRunning(t *testing.T) {
resp, err := http.Get(baseURL)
if resp.StatusCode != 200 || err != nil {
fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
t.Logf("got %v, err = %v\n", resp.StatusCode, err)
t.Fatal(err)
}
}
@@ -52,7 +51,7 @@ func TestServerRunning(t *testing.T) {
func TestBadRequest(t *testing.T) {
resp, err := http.Get(baseURL + "doesnotexist.html")
if resp.StatusCode != 404 || err != nil {
fmt.Printf("got %v, err = %v\n", resp.StatusCode, err)
t.Logf("got %v, err = %v\n", resp.StatusCode, err)
t.Fatal("request for non existent file didn't fail properly")
}
}
@@ -65,8 +64,8 @@ func TestSimpleFetch(t *testing.T) {
}
desired := "<html><head></head><body></body></html>\n"
if body != desired {
fmt.Printf(" Got: '%v'\n", body)
fmt.Printf("Wanted: '%v'\n", desired)
t.Logf(" Got: '%v'\n", body)
t.Logf("Wanted: '%v'\n", desired)
t.Error("fetch succeeded, but content mismatched")
}
}
@@ -74,22 +73,40 @@ func TestSimpleFetch(t *testing.T) {
// TestSimpleMap figures out the site map for the site in baseURL
func TestSimpleMap(t *testing.T) {
pages := docrawl(baseURL)
t.Log("pages = ", pages)
if len(pages) != 1 {
t.Logf("got %v, wanted %v\n", len(pages), 1)
t.Fatal("got wrong number of pages")
}
if pages[0].URL != baseURL {
if pages[0].URL.String() != baseURL {
t.Error("page URL is invalid")
}
if len(pages[0].Assets) != 1 {
t.Fatal("got wrong number of assets")
}
if pages[0].Assets[0] != baseURL+"/assets/image.png" {
t.Error("asset name is incorrect")
}
if len(pages[0].Links) != 1 {
if len(pages[0].Children) != 2 {
t.Logf("got %v, wanted %v\n", len(pages[0].Children), 1)
t.Fatal("got wrong number of links")
}
if pages[0].Links[0] != baseURL+"/about.html" {
if pages[0].Children[0].URL.String() != baseURL+"about.html" {
t.Logf(" Got: %q\n", pages[0].Children[0].URL.String())
t.Logf("Wanted: %q\n", baseURL+"about.html")
t.Error("link name is incorrect")
}
if pages[0].Children[1].URL.String() != baseURL+"assets/image.png" {
t.Logf(" Got: %q\n", pages[0].Children[1].URL.String())
t.Logf("Wanted: %q\n", baseURL+"assets/image.png")
t.Error("link name is incorrect")
}
}
// TestHeaderFetching tests fetch_filetype() to see if we are getting expected results
func TestHeaderFetching(t *testing.T) {
if ft, err := fetch_filetype(baseURL); err != nil || ft != "text/html" {
t.Error("problem fetching filetype")
}
if ft, err := fetch_filetype(baseURL + "about.html"); err != nil || ft != "text/html" {
t.Error("problem fetching filetype")
}
if ft, err := fetch_filetype(baseURL + "assets/image.png"); err != nil || ft != "text/plain" {
t.Logf("got %q, wanted %q", ft, "text/plain")
t.Error("problem fetching filetype")
}
}
src/docrawler/fetch.go
+31
View File
@@ -1,12 +1,43 @@
package main
import (
"errors"
"io/ioutil"
"mime"
"net/http"
)
var (
ErrContentTypeNotFound = errors.New("no Content-Type header found")
)
func fetch_filetype(url string) (string, error) {
resp, err := http.Head(url)
if err != nil {
return "", err
}
// pull the content type out of the http header
contentType, ok := resp.Header["Content-Type"]
if !ok {
// no Content-Type header found
return "", ErrContentTypeNotFound
}
// parse mime type
mediatype, _, err := mime.ParseMediaType(contentType[0])
if err != nil {
return "", err
}
return mediatype, nil
}
// fetch takes a URL, GETs it, and returns the body as a string
func fetch(url string) (string, error) {
// figure out the file type
//filetype, err := fetch_filetype(url)
// GET the url
resp, err := http.Get(url)
if err != nil {
src/docrawler/testsite/about.html
View File
Whitespace-only changes.
src/docrawler/testsite/image.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 938 B

src/docrawler/url_test.go
+5 -5
View File
@@ -1,7 +1,6 @@
package main
import (
"fmt"
"net/url"
"testing"
)
@@ -14,8 +13,8 @@ func TestURLCleaner(t *testing.T) {
}
cleanURL(u)
if u.Host != "somehost.com:8765" {
fmt.Printf(" Got: %q\n", u.Host)
fmt.Printf("Wanted: %q\n", "somehost.com:8765")
t.Logf(" Got: %q\n", u.Host)
t.Logf("Wanted: %q\n", "somehost.com:8765")
t.Error("host extraction failed")
}
if u.String() != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
@@ -41,8 +40,8 @@ func doResolveTest(t *testing.T, wanted, referrer, current string) {
t.Error("resolveURL failed (probably parsing)")
}
if u.String() != wanted {
fmt.Printf(" Got: %q\n", u)
fmt.Printf("Wanted: %q\n", wanted)
t.Logf(" Got: %q\n", u)
t.Logf("Wanted: %q\n", wanted)
t.Error("url resolution with base URL failed")
}
}
@@ -51,4 +50,5 @@ func TestURLResolution(t *testing.T) {
doResolveTest(t, "http://base.com/some/crazy/path/test.png", "http://BaSe.CoM/some/crazy/path/index.html?parm=x&blah=foo", "test.png")
doResolveTest(t, "http://S:D@base.com:1/test.png", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "/test.png")
doResolveTest(t, "http://another.com/blah.html", "http://S:D@BaSe.CoM:1/some/crazy/path/index.html?parm=x&blah=foo", "http://aNOThER.CoM/blah.html")
doResolveTest(t, "http://empty.com/blah.html", "", "http://empty.com/blah.html")
}