flow: Closed <feature> 'extract_titles_from_pages'.

Makefile

@@ -31,7 +31,7 @@ vet:
 	go vet ./src/...
 
 test:
-	go test ./src/... | ./util/testfilter.sh
+	go test -v ./src/... | ./util/testfilter.sh
 
 autotest:
 	fswatch -or ./src | \

src/docrawler/parse.go

@@ -3,19 +3,28 @@ package main
 import (
 	"regexp"
 	"sort"
+	"strings"
 )
 
 // a regexp which captures src/href/xhref="..." from text
-var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC nice
+var reURL = regexp.MustCompile(`(src|href|xhref)\s*=\s*"([^"]+)"`) // TODO make RFC compliant
+var reTitle = regexp.MustCompile(`<\s*title\s*>([^<]*)`)
 
-// parse takes a string and attempts to parse any html links out of it,
+// parse takes a string and attempts to parse any html title and all links out of it,
 // and returns a sorted slice of the captures found
-func parse(s string) ([]string, error) {
-	// find all of the matches (submatch = include capture groups)
+func parse(s string) (string, []string, error) {
+	// find the title, or default to empty
+	title := ""
+	titleMatches := reTitle.FindStringSubmatch(s)
+	if titleMatches != nil {
+		title = strings.TrimSpace(titleMatches[1])
+	}
+
+	// find all of the URL matches (submatch = include capture groups)
 	results := []string{}
 	for _, match := range reURL.FindAllStringSubmatch(s, -1) {
 		results = append(results, match[2])
 	}
 	sort.Strings(results)
-	return results, nil
+	return title, results, nil
 }

src/docrawler/parse_test.go

@@ -4,20 +4,24 @@ import (
 	"testing"
 )
 
-// TestSimpleParse verifies that we can extract URLs (etc.) from a known document
+// TestSimpleParse verifies that we can extract the page Title and all URLs (etc.) from a known document
 func TestSimpleParse(t *testing.T) {
 	doc := `<html>
 	<head>
+		<title> Test Page  </title>
 	</head>
 	<body>
 		<img src="/assets/image.png"/>
 		<a href="/about.html">
 	</body>
 </html>`
-	matches, err := parse(doc)
+	title, matches, err := parse(doc)
 	if err != nil {
 		t.Error(err)
 	}
+	if title != "Test Page" {
+		t.Error("got wrong title")
+	}
 	if len(matches) != 2 {
 		t.Error("invalid number of matches in parse")
 	}