Repository: tmc/langchaingo
Pull request #158 (2023-06-28 20:33:24)

document loaders: add html loader
Introducing HTML document loader (including parsing and sanitizing).

### PR Checklist

- [X] Read the [Contributing documentation](https://github.com/tmc/langchaingo/blob/main/CONTRIBUTING.md).
- [x] Read the [Code of conduct documentation](https://github.com/tmc/langchaingo/blob/main/CODE_OF_CONDUCT.md).
- [x] Name your Pull Request title clearly, concisely, and prefixed with the name of the primarily affected package you changed according to [Good commit messages](https://go.dev/doc/contribute#commit_messages) (such as `memory: add interfaces for X, Y` or `util: add whizzbang helpers`).
- [x] Check that there isn't already a PR that solves the problem the same way to avoid creating a duplicate.
- [x] Provide a description in this PR that addresses **what** the PR is solving, or reference the issue that it solves (e.g. `Fixes #123`).
- [x] Describes the source of new concepts.
- [x] References existing implementations as appropriate.
- [x] Contains test coverage for new functions.
- [x] Passes all [`golangci-lint`](https://golangci-lint.run/) checks.



Patch:
diff --git a/docs/parity_matrix.md b/docs/parity_matrix.md
index e122a5c6..64c55a98 100644
--- a/docs/parity_matrix.md
+++ b/docs/parity_matrix.md
@@ -89,7 +89,7 @@ Please note that this page lists the current state of the LangChain Go project,
 | Retry                       | ❌     |
 | Guardrail                   | ❌     |
 
-##Document Loaders
+## Document Loaders
 
 | Feature                         | Status |
 |---------------------------------|--------|
@@ -125,7 +125,7 @@ Please note that this page lists the current state of the LangChain Go project,
 | Google Drive                    | ❌      |
 | Gutenberg Books                 | ❌      |
 | HN                              | ❌      |
-| HTML                            | ❌      |
+| HTML                            | ✅      |
 | HTML BS                         | ❌      |
 | Hugging Face Dataset            | ❌      |
 | iFixit                          | ❌      |
diff --git a/documentloaders/html.go b/documentloaders/html.go
new file mode 100644
index 00000000..5c6b6d99
--- /dev/null
+++ b/documentloaders/html.go
@@ -0,0 +1,59 @@
+package documentloaders
+
+import (
+	"context"
+	"io"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/tmc/langchaingo/schema"
+	"github.com/tmc/langchaingo/textsplitter"
+)
+
+// HTML loads parses and sanitizes html content from an io.Reader.
+type HTML struct {
+	r io.Reader
+}
+
+var _ Loader = HTML{}
+
+// NewHTML creates a new html loader with an io.Reader.
+func NewHTML(r io.Reader) HTML {
+	return HTML{r}
+}
+
+// Load reads from the io.Reader and returns a single document with the data.
+func (h HTML) Load(_ context.Context) ([]schema.Document, error) {
+	doc, err := goquery.NewDocumentFromReader(h.r)
+	if err != nil {
+		return nil, err
+	}
+
+	var sel *goquery.Selection
+	if doc.Has("body") != nil {
+		sel = doc.Find("body").Contents()
+	} else {
+		sel = doc.Contents()
+	}
+
+	sanitized := bluemonday.UGCPolicy().Sanitize(sel.Text())
+	pagecontent := strings.TrimSpace(sanitized)
+
+	return []schema.Document{
+		{
+			PageContent: pagecontent,
+			Metadata:    map[string]any{},
+		},
+	}, nil
+}
+
+// LoadAndSplit reads text data from the io.Reader and splits it into multiple
+// documents using a text splitter.
+func (h HTML) LoadAndSplit(ctx context.Context, splitter textsplitter.TextSplitter) ([]schema.Document, error) {
+	docs, err := h.Load(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return textsplitter.SplitDocuments(splitter, docs)
+}
diff --git a/go.mod b/go.mod
index 4f0791b1..9c932d9d 100644
--- a/go.mod
+++ b/go.mod
@@ -15,10 +15,11 @@ require (
 	cloud.google.com/go/longrunning v0.4.2 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
@@ -34,6 +35,7 @@ require (
 	github.com/google/s2a-go v0.1.3 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
@@ -66,6 +68,7 @@ require (
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/google/go-cmp v0.5.9
 	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/microcosm-cc/bluemonday v1.0.24
 	github.com/pinecone-io/go-pinecone v0.3.0
 	github.com/pkoukk/tiktoken-go v0.1.2
 	github.com/weaviate/weaviate v1.19.0
diff --git a/go.sum b/go.sum
index 65fcab4d..4e9d4ae3 100644
--- a/go.sum
+++ b/go.sum
@@ -54,16 +54,18 @@ github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBa
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -219,6 +221,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
 github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2 h1:I/pwhnUln5wbMnTyRbzswA0/JxpK8sZj0aUfI3TV1So=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.2/go.mod h1:lsuH8kb4GlMdSlI4alNIBBSAt5CHJtg3i+0WuN9J5YM=
@@ -256,6 +260,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
+github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
 github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -415,8 +421,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=


Test patch:
diff --git a/documentloaders/html_test.go b/documentloaders/html_test.go
new file mode 100644
index 00000000..47885fea
--- /dev/null
+++ b/documentloaders/html_test.go
@@ -0,0 +1,50 @@
+package documentloaders
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTMLLoader(t *testing.T) {
+	t.Parallel()
+	file, err := os.Open("./testdata/test.html")
+	assert.NoError(t, err)
+
+	loader := NewHTML(file)
+
+	docs, err := loader.Load(context.Background())
+	require.NoError(t, err)
+	require.Len(t, docs, 1)
+
+	content := docs[0].PageContent
+	expected := []string{
+		"The content",
+		"goes here",
+		"and here",
+	}
+	notexpected := []string{
+		"console.log(",
+		"<title>langchaingo html example",
+		"</title>",
+		"<footer>",
+		"XSS1",
+		"onmouseover",
+	}
+
+	assert.Contains(t, content, expected[0])
+	assert.Contains(t, content, expected[1])
+	assert.Contains(t, content, expected[2])
+	assert.NotContains(t, content, notexpected[0])
+	assert.NotContains(t, content, notexpected[1])
+	assert.NotContains(t, content, notexpected[2])
+	assert.NotContains(t, content, notexpected[3])
+	assert.NotContains(t, content, notexpected[4])
+	assert.NotContains(t, content, notexpected[5])
+
+	expectedMetadata := map[string]any{}
+	assert.Equal(t, docs[0].Metadata, expectedMetadata)
+}
diff --git a/documentloaders/testdata/test.html b/documentloaders/testdata/test.html
new file mode 100644
index 00000000..34a69172
--- /dev/null
+++ b/documentloaders/testdata/test.html
@@ -0,0 +1,16 @@
+<html>
+  <head>
+    <title>langchaingo html example</title>
+    <script>console.log("langchaingo rocks")</script>
+  </head>
+  <body>
+    <h1>The content</h1>
+    <p><a href="javascript:alert('XSS1')" onmouseover="alert('XSS2')">goes <span>here</span></a></p>
+    <p></p>
+    <footer>
+      <ul>
+        <li> and here</li>
+      </ul>
+    </footer>
+  </body>
+</html>