From 4261e84eb8a3730c7716f0e1645e6d70249fd277 Mon Sep 17 00:00:00 2001
From: Yoomitech <301026915+Yoomitech@users.noreply.github.com>
Date: Sat, 25 Jul 2026 08:12:39 +0200
Subject: [PATCH] fix(extract): honor HTML base href for relative URLs
---
extract/content.go | 22 ++++++++---
extract/extractor.go | 10 ++++-
extract/extractor_test.go | 80 +++++++++++++++++++++++++++++++++++++++
extract/metadata.go | 36 ++++++++++++++++++
4 files changed, 140 insertions(+), 8 deletions(-)
diff --git a/extract/content.go b/extract/content.go
index c8fc852..c19dbf0 100644
--- a/extract/content.go
+++ b/extract/content.go
@@ -40,9 +40,19 @@ const minCleanTextRunes = 250
// comes back too thin for a page that clearly had content, it transparently
// falls back to full-body extraction. When clean is false it skips article
// detection entirely and converts the whole readable .
-func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult, error) {
+// extractContent keeps the fetched document URL separate from the effective
+// HTML link base. This matters for pages using .
+func extractContent(
+ htmlBytes []byte,
+ documentURL string,
+ linkBaseURL string,
+ clean bool,
+) (contentResult, error) {
+ if strings.TrimSpace(linkBaseURL) == "" {
+ linkBaseURL = documentURL
+ }
if !clean {
- return extractFullBody(htmlBytes, baseURL)
+ return extractFullBody(htmlBytes, linkBaseURL)
}
var out contentResult
@@ -54,7 +64,7 @@ func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult
IncludeLinks: true,
Deduplicate: true,
}
- if parsed, err := url.Parse(baseURL); err == nil {
+ if parsed, err := url.Parse(documentURL); err == nil {
opts.OriginalURL = parsed
}
extracted, err := trafilatura.Extract(bytes.NewReader(htmlBytes), opts)
@@ -62,7 +72,7 @@ func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult
return out, err
}
if extracted == nil || extracted.ContentNode == nil {
- return extractFullBody(htmlBytes, baseURL)
+ return extractFullBody(htmlBytes, linkBaseURL)
}
var htmlBuf bytes.Buffer
@@ -75,7 +85,7 @@ func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult
out.Description = strings.TrimSpace(extracted.Metadata.Description)
out.Lang = strings.TrimSpace(extracted.Metadata.Language)
- markdown, err := htmlToMarkdown(out.HTML, baseURL)
+ markdown, err := htmlToMarkdown(out.HTML, linkBaseURL)
if err != nil {
return out, err
}
@@ -84,7 +94,7 @@ func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult
// trafilatura was too aggressive: the cleaned article is near-empty but the
// raw page had real visible text. Prefer the fuller readable-body pass.
if len([]rune(out.Text)) < minCleanTextRunes {
- if full, ferr := extractFullBody(htmlBytes, baseURL); ferr == nil &&
+ if full, ferr := extractFullBody(htmlBytes, linkBaseURL); ferr == nil &&
len([]rune(full.Text)) > len([]rune(out.Text)) {
// Keep trafilatura's metadata (title/description/lang) when present;
// it is usually cleaner than what we derive from the full body.
diff --git a/extract/extractor.go b/extract/extractor.go
index cec0580..69d8a9a 100644
--- a/extract/extractor.go
+++ b/extract/extractor.go
@@ -135,8 +135,14 @@ func buildResult(req ExtractRequest, resp *FetchResponse, mode string, startedAt
if err != nil {
return nil, err
}
- metadata := parseMetadata(doc, req.URL)
- content, contentErr := extractContent(body, req.URL, !req.FullPage)
+ linkBaseURL := effectiveBaseURL(doc, req.URL)
+ metadata := parseMetadata(doc, linkBaseURL)
+ content, contentErr := extractContent(
+ body,
+ req.URL,
+ linkBaseURL,
+ !req.FullPage,
+ )
result := &ExtractResult{
URL: req.URL,
Title: firstNonEmpty(content.Title, metadata.Title),
diff --git a/extract/extractor_test.go b/extract/extractor_test.go
index 316aa41..71527ac 100644
--- a/extract/extractor_test.go
+++ b/extract/extractor_test.go
@@ -305,6 +305,86 @@ func TestParseMetadataRichTags(t *testing.T) {
}
}
+func TestParseMetadataHonorsBaseHref(t *testing.T) {
+ doc, err := documentFromString(`
+
+
+
+
+
+
+ Die Weisheit des Schamanen
+
+`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ documentURL := "https://example.com/de/ein_wolf_kommt_zu_wort.html"
+ linkBaseURL := effectiveBaseURL(doc, documentURL)
+
+ if linkBaseURL != "https://example.com/" {
+ t.Fatalf("effective base URL = %q, want %q", linkBaseURL, "https://example.com/")
+ }
+
+ meta := parseMetadata(doc, linkBaseURL)
+
+ if meta.Canonical != "https://example.com/de/canonical.html" {
+ t.Fatalf("canonical = %q", meta.Canonical)
+ }
+
+ if len(meta.Links) != 1 {
+ t.Fatalf("link count = %d, want 1", len(meta.Links))
+ }
+
+ want := "https://example.com/de/die_weisheit_des_schamanen.html"
+ if meta.Links[0].URL != want {
+ t.Fatalf("resolved link = %q, want %q", meta.Links[0].URL, want)
+ }
+
+ if strings.Contains(meta.Links[0].URL, "/de/de/") {
+ t.Fatalf("resolved link contains duplicated language path: %q", meta.Links[0].URL)
+ }
+}
+
+func TestExtractContentHonorsBaseHref(t *testing.T) {
+ htmlBytes := []byte(`
+
+
+ Die Weisheit des Schamanen
+ 
+
+`)
+
+ result, err := extractContent(
+ htmlBytes,
+ "https://example.com/de/ein_wolf_kommt_zu_wort.html",
+ "https://example.com/",
+ false,
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !strings.Contains(
+ result.Markdown,
+ "https://example.com/de/die_weisheit_des_schamanen.html",
+ ) {
+ t.Fatalf("markdown link was not resolved against : %q", result.Markdown)
+ }
+
+ if !strings.Contains(
+ result.Markdown,
+ "https://example.com/de/images/example.png",
+ ) {
+ t.Fatalf("markdown image was not resolved against : %q", result.Markdown)
+ }
+
+ if strings.Contains(result.Markdown, "/de/de/") {
+ t.Fatalf("markdown contains duplicated language path: %q", result.Markdown)
+ }
+}
+
func documentFromString(raw string) (*goquery.Document, error) {
return goquery.NewDocumentFromReader(strings.NewReader(raw))
}
diff --git a/extract/metadata.go b/extract/metadata.go
index e34c1ca..37450ff 100644
--- a/extract/metadata.go
+++ b/extract/metadata.go
@@ -20,6 +20,42 @@ type pageMetadata struct {
Links []Link
}
+// effectiveBaseURL returns the URL base defined by the document's first
+// element. When no usable HTTP(S) base exists, the original
+// document URL remains the base.
+//
+// Relative links in HTML must be resolved against , not blindly
+// against the fetched document URL.
+func effectiveBaseURL(doc *goquery.Document, documentURL string) string {
+ documentURL = strings.TrimSpace(documentURL)
+ if doc == nil || documentURL == "" {
+ return documentURL
+ }
+
+ baseHref, ok := doc.Find("base[href]").First().Attr("href")
+ baseHref = strings.TrimSpace(baseHref)
+ if !ok || baseHref == "" {
+ return documentURL
+ }
+
+ resolved := resolveURL(documentURL, baseHref)
+ if resolved == "" {
+ return documentURL
+ }
+
+ parsed, err := url.Parse(resolved)
+ if err != nil {
+ return documentURL
+ }
+
+ // A document base for fetched web content must remain an HTTP(S) URL.
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return documentURL
+ }
+
+ return parsed.String()
+}
+
func parseMetadata(doc *goquery.Document, baseURL string) pageMetadata {
var meta pageMetadata
if doc == nil {