diff --git a/Makefile b/Makefile index e4cdaac..aefe125 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ vet: go vet ./src/... test: - go test ./src/... | ./util/testfilter.sh + go test -v ./src/... | ./util/testfilter.sh autotest: fswatch -or ./src | \ diff --git a/src/docrawler/parse.go b/src/docrawler/parse.go index 30e678a..e03a667 100644 --- a/src/docrawler/parse.go +++ b/src/docrawler/parse.go @@ -3,19 +3,28 @@ package main import ( "regexp" "sort" + "strings" ) // a regexp which captures src/href/xhref="..." from text -var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice +var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant +var reTitle = regexp.MustCompile(`<\s*title\s*>([^<]*)`) -// parse takes a string and attempts to parse any html links out of it, +// parse takes a string and attempts to parse any html title and all links out of it, // and returns a sorted slice of the captures found -func parse(s string) ([]string, error) { - // find all of the matches (submatch = include capture groups) +func parse(s string) (string, []string, error) { + // find the title, or default to empty + title := "" + titleMatches := reTitle.FindStringSubmatch(s) + if titleMatches != nil { + title = strings.TrimSpace(titleMatches[1]) + } + + // find all of the URL matches (submatch = include capture groups) results := []string{} for _, match := range reURL.FindAllStringSubmatch(s, -1) { results = append(results, match[2]) } sort.Strings(results) - return results, nil + return title, results, nil } diff --git a/src/docrawler/parse_test.go b/src/docrawler/parse_test.go index 738f2c4..eae5883 100644 --- a/src/docrawler/parse_test.go +++ b/src/docrawler/parse_test.go @@ -4,20 +4,24 @@ import ( "testing" ) -// TestSimpleParse verifies that we can extract URLs (etc.) from a known document +// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document func TestSimpleParse(t *testing.T) { doc := ` + Test Page ` - matches, err := parse(doc) + title, matches, err := parse(doc) if err != nil { t.Error(err) } + if title != "Test Page" { + t.Error("got wrong title") + } if len(matches) != 2 { t.Error("invalid number of matches in parse") }