diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 288a83fc1855..34df41f26e9e 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -34,7 +34,7 @@ import type { JsonValue } from 'type-fest';
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
-import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
+import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
/**
* Default mime types, which HttpScraper supports.
@@ -640,6 +640,15 @@ export class HttpCrawler<
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
} else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
+ if (!charset && !this.forceResponseEncoding) {
+ const rawBytes = Buffer.from(await response.arrayBuffer());
+ const metaCharset = extractCharsetFromHtmlBytes(rawBytes);
+ const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8';
+ const body = iconv.encodingExists(charsetToUse)
+ ? iconv.decode(rawBytes, charsetToUse)
+ : rawBytes.toString('utf8');
+ return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body };
+ }
return { response, contentType, body: await reencodedResponse.text() };
} else {
const body = Buffer.from(await reencodedResponse.bytes());
diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.ts
index 4111da0ea169..3631571f1646 100644
--- a/packages/http-crawler/src/internals/utils.ts
+++ b/packages/http-crawler/src/internals/utils.ts
@@ -59,6 +59,18 @@ export function processHttpRequestOptions({
return { ...request, body, url, headers };
}
+/**
+ * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
+ * declared via `` or ``.
+ * This implements a simplified version of the HTML spec's byte-stream prescan algorithm.
+ */
+export function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined {
+ // latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable
+ const prescan = bytes.subarray(0, 1024).toString('latin1');
+ const match = /]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan);
+ return match?.[1];
+}
+
/**
* Gets parsed content type from response object
* @param response HTTP response object
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 2605e3ebd4ab..b7e7aa732492 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -695,6 +695,22 @@ describe('CheerioCrawler', () => {
expect(await response.text()).toBe(html);
});
+ test('via http-equiv meta tag when no charset in HTTP header', async () => {
+ let context: CheerioCrawlingContext | null = null;
+
+ const crawler = new CheerioCrawler({
+ requestHandler: (ctx) => {
+ context = ctx;
+ },
+ });
+
+ await crawler.run([`${serverAddress}/special/meta-charset`]);
+
+ context = context as unknown as CheerioCrawlingContext;
+ expect(context?.body).toContain('Žluťoučký kůň');
+ expect(context?.$('body').text()).toContain('Žluťoučký kůň');
+ });
+
test('Cheerio decodes html entities', async () => {
let context: CheerioCrawlingContext | null = null;
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 198c03f221d5..894f9aa869df 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -4,6 +4,7 @@ import { Readable } from 'node:stream';
import { HttpCrawler, SessionPool } from '@crawlee/http';
import { ResponseWithUrl } from '@crawlee/http-client';
+import iconv from 'iconv-lite';
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
const router = new Map();
@@ -60,6 +61,20 @@ router.set('/403-with-octet-stream', (req, res) => {
res.end();
});
+router.set('/meta-charset', (req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `${text}`;
+ res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+ res.end(iconv.encode(html, 'windows-1250'));
+});
+
+router.set('/meta-charset-html5', (req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `${text}`;
+ res.setHeader('content-type', 'text/html'); // no charset in HTTP header
+ res.end(iconv.encode(html, 'windows-1250'));
+});
+
let server: http.Server;
let url: string;
@@ -208,6 +223,36 @@ test('invalid content type defaults to octet-stream', async () => {
]);
});
+test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => {
+ const results: string[] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ requestHandler: ({ body }) => {
+ results.push(body as string);
+ },
+ });
+
+ await crawler.run([`${url}/meta-charset`]);
+
+ expect(results[0]).toContain('Žluťoučký kůň');
+});
+
+test('decodes charset from HTML5 meta charset attribute when absent in HTTP header', async () => {
+ const results: string[] = [];
+
+ const crawler = new HttpCrawler({
+ maxRequestRetries: 0,
+ requestHandler: ({ body }) => {
+ results.push(body as string);
+ },
+ });
+
+ await crawler.run([`${url}/meta-charset-html5`]);
+
+ expect(results[0]).toContain('Žluťoučký kůň');
+});
+
test('handles cookies from redirects', async () => {
const results: string[] = [];
diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
index e1cc1c1cb5cc..ad9d0d215f0c 100644
--- a/test/shared/_helper.ts
+++ b/test/shared/_helper.ts
@@ -7,6 +7,7 @@ import bodyParser from 'body-parser';
import { entries } from 'crawlee';
import type { Application } from 'express';
import express from 'express';
+import iconv from 'iconv-lite';
export const startExpressAppPromise = async (app: Application, port: number) => {
return new Promise((resolve) => {
@@ -332,6 +333,13 @@ export async function runExampleComServer(): Promise<[Server, number]> {
res.type('html').send('"<>"<>');
});
+ special.get('/meta-charset', (_req, res) => {
+ const text = 'Žluťoučký kůň';
+ const html = `${text}`;
+ res.setHeader('content-type', 'text/html');
+ res.end(iconv.encode(html, 'windows-1250'));
+ });
+
special.get('/set-cookie', (req, res) => {
const cookieName = (req.query.name as string) || 'testCookie';
const cookieValue = (req.query.value as string) || 'testValue';