Skip to content

fix: read content encoding hints from <meta> tags #3780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
barjin merged 3 commits into from
Jun 30, 2026
Merged

fix: read content encoding hints from <meta> tags #3780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cd50e0e
fix: read content encoding hints from `<meta>` tags
barjin Jun 24, 2026
367222e
chore: fix types
barjin Jun 24, 2026
08da78b
chore: fix lint errors
barjin Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion packages/http-crawler/src/internals/http-crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import type { JsonValue } from 'type-fest';

import { addTimeoutToPromise, tryCancel } from '@apify/timeout';

import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';

/**
* Default mime types, which HttpScraper supports.
Expand Down Expand Up @@ -640,6 +640,15 @@ export class HttpCrawler<
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
if (!charset && !this.forceResponseEncoding) {
const rawBytes = Buffer.from(await response.arrayBuffer());
const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
const body = iconv.encodingExists(charsetToUse)
? iconv.decode(rawBytes, charsetToUse)
: rawBytes.toString('utf8');
return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body };
}
return { response, contentType, body: await reencodedResponse.text() };
} else {
const body = Buffer.from(await reencodedResponse.bytes());
Expand Down
12 changes: 12 additions & 0 deletions packages/http-crawler/src/internals/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,18 @@ export function processHttpRequestOptions({
return { ...request, body, url, headers };
}

/**
* Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
* declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
* This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
*/
export function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined {
// latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable
const prescan = bytes.subarray(0, 1024).toString('latin1');

@janbuchar janbuchar Jun 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure that 1024 bytes will do it?

@barjin barjin Jun 30, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See 13.2.3.2 Determining the character encoding paragraph 5:

Optionally, prescan the byte stream to determine its encoding, with the end condition being when the user agent decides that scanning further bytes would not be efficient. User agents are encouraged to only prescan the first 1024 bytes. User agents may decide that scanning any bytes is not efficient, in which case these substeps are entirely skipped.

@janbuchar janbuchar Jun 30, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, lawyered. Approving.

const match = /<meta[^>]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan);
return match?.[1];
}

/**
* Gets parsed content type from response object
* @param response HTTP response object
Expand Down
16 changes: 16 additions & 0 deletions test/core/crawlers/cheerio_crawler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,22 @@ describe('CheerioCrawler', () => {
expect(await response.text()).toBe(html);
});

test('via http-equiv meta tag when no charset in HTTP header', async () => {
let context: CheerioCrawlingContext | null = null;

const crawler = new CheerioCrawler({
requestHandler: (ctx) => {
context = ctx;
},
});

await crawler.run([`${serverAddress}/special/meta-charset`]);

context = context as unknown as CheerioCrawlingContext;
expect(context?.body).toContain('Žluťoučký kůň');
expect(context?.$('body').text()).toContain('Žluťoučký kůň');
});

test('Cheerio decodes html entities', async () => {
let context: CheerioCrawlingContext | null = null;

Expand Down
45 changes: 45 additions & 0 deletions test/core/crawlers/http_crawler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { Readable } from 'node:stream';

import { HttpCrawler, SessionPool } from '@crawlee/http';
import { ResponseWithUrl } from '@crawlee/http-client';
import iconv from 'iconv-lite';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';

const router = new Map<string, http.RequestListener>();
Expand Down Expand Up @@ -60,6 +61,20 @@ router.set('/403-with-octet-stream', (req, res) => {
res.end();
});

router.set('/meta-charset', (req, res) => {
const text = 'Žluťoučký kůň';
const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
res.setHeader('content-type', 'text/html'); // no charset in HTTP header
res.end(iconv.encode(html, 'windows-1250'));
});

router.set('/meta-charset-html5', (req, res) => {
const text = 'Žluťoučký kůň';
const html = `<html><head><meta charset="windows-1250"></head><body>${text}</body></html>`;
res.setHeader('content-type', 'text/html'); // no charset in HTTP header
res.end(iconv.encode(html, 'windows-1250'));
});

let server: http.Server;
let url: string;

Expand Down Expand Up @@ -208,6 +223,36 @@ test('invalid content type defaults to octet-stream', async () => {
]);
});

test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => {
const results: string[] = [];

const crawler = new HttpCrawler({
maxRequestRetries: 0,
requestHandler: ({ body }) => {
results.push(body as string);
},
});

await crawler.run([`${url}/meta-charset`]);

expect(results[0]).toContain('Žluťoučký kůň');
});

test('decodes charset from HTML5 meta charset attribute when absent in HTTP header', async () => {
const results: string[] = [];

const crawler = new HttpCrawler({
maxRequestRetries: 0,
requestHandler: ({ body }) => {
results.push(body as string);
},
});

await crawler.run([`${url}/meta-charset-html5`]);

expect(results[0]).toContain('Žluťoučký kůň');
});

test('handles cookies from redirects', async () => {
const results: string[] = [];

Expand Down
8 changes: 8 additions & 0 deletions test/shared/_helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import bodyParser from 'body-parser';
import { entries } from 'crawlee';
import type { Application } from 'express';
import express from 'express';
import iconv from 'iconv-lite';

export const startExpressAppPromise = async (app: Application, port: number) => {
return new Promise<Server>((resolve) => {
Expand Down Expand Up @@ -332,6 +333,13 @@ export async function runExampleComServer(): Promise<[Server, number]> {
res.type('html').send('&quot;&lt;&gt;"<>');
});

special.get('/meta-charset', (_req, res) => {
const text = 'Žluťoučký kůň';
const html = `<html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1250"></head><body>${text}</body></html>`;
res.setHeader('content-type', 'text/html');
res.end(iconv.encode(html, 'windows-1250'));
});

special.get('/set-cookie', (req, res) => {
const cookieName = (req.query.name as string) || 'testCookie';
const cookieValue = (req.query.value as string) || 'testValue';
Expand Down
Loading