diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts index 288a83fc1855..34df41f26e9e 100644 --- a/packages/http-crawler/src/internals/http-crawler.ts +++ b/packages/http-crawler/src/internals/http-crawler.ts @@ -34,7 +34,7 @@ import type { JsonValue } from 'type-fest'; import { addTimeoutToPromise, tryCancel } from '@apify/timeout'; -import { parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js'; +import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js'; /** * Default mime types, which HttpScraper supports. @@ -640,6 +640,15 @@ export class HttpCrawler< // It's not a JSON, so it's probably some text. Get the first 100 chars of it. throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`); } else if (HTML_AND_XML_MIME_TYPES.includes(type)) { + if (!charset && !this.forceResponseEncoding) { + const rawBytes = Buffer.from(await response.arrayBuffer()); + const metaCharset = extractCharsetFromHtmlBytes(rawBytes); + const charsetToUse = metaCharset ?? this.suggestResponseEncoding ?? 'utf-8'; + const body = iconv.encodingExists(charsetToUse) + ? iconv.decode(rawBytes, charsetToUse) + : rawBytes.toString('utf8'); + return { response, contentType: { type, encoding: 'utf-8' as BufferEncoding }, body }; + } return { response, contentType, body: await reencodedResponse.text() }; } else { const body = Buffer.from(await reencodedResponse.bytes()); diff --git a/packages/http-crawler/src/internals/utils.ts b/packages/http-crawler/src/internals/utils.ts index 4111da0ea169..3631571f1646 100644 --- a/packages/http-crawler/src/internals/utils.ts +++ b/packages/http-crawler/src/internals/utils.ts @@ -59,6 +59,18 @@ export function processHttpRequestOptions({ return { ...request, body, url, headers }; } +/** + * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset + * declared via `` or ``. + * This implements a simplified version of the HTML spec's byte-stream prescan algorithm. + */ +export function extractCharsetFromHtmlBytes(bytes: Buffer): string | undefined { + // latin1 preserves byte values for ASCII-compatible encodings, making the meta tags readable + const prescan = bytes.subarray(0, 1024).toString('latin1'); + const match = /]+\bcharset\s*=\s*["']?\s*([^"'\s;>]+)/i.exec(prescan); + return match?.[1]; +} + /** * Gets parsed content type from response object * @param response HTTP response object diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts index 2605e3ebd4ab..b7e7aa732492 100644 --- a/test/core/crawlers/cheerio_crawler.test.ts +++ b/test/core/crawlers/cheerio_crawler.test.ts @@ -695,6 +695,22 @@ describe('CheerioCrawler', () => { expect(await response.text()).toBe(html); }); + test('via http-equiv meta tag when no charset in HTTP header', async () => { + let context: CheerioCrawlingContext | null = null; + + const crawler = new CheerioCrawler({ + requestHandler: (ctx) => { + context = ctx; + }, + }); + + await crawler.run([`${serverAddress}/special/meta-charset`]); + + context = context as unknown as CheerioCrawlingContext; + expect(context?.body).toContain('Žluťoučký kůň'); + expect(context?.$('body').text()).toContain('Žluťoučký kůň'); + }); + test('Cheerio decodes html entities', async () => { let context: CheerioCrawlingContext | null = null; diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts index 198c03f221d5..894f9aa869df 100644 --- a/test/core/crawlers/http_crawler.test.ts +++ b/test/core/crawlers/http_crawler.test.ts @@ -4,6 +4,7 @@ import { Readable } from 'node:stream'; import { HttpCrawler, SessionPool } from '@crawlee/http'; import { ResponseWithUrl } from '@crawlee/http-client'; +import iconv from 'iconv-lite'; import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js'; const router = new Map(); @@ -60,6 +61,20 @@ router.set('/403-with-octet-stream', (req, res) => { res.end(); }); +router.set('/meta-charset', (req, res) => { + const text = 'Žluťoučký kůň'; + const html = `${text}`; + res.setHeader('content-type', 'text/html'); // no charset in HTTP header + res.end(iconv.encode(html, 'windows-1250')); +}); + +router.set('/meta-charset-html5', (req, res) => { + const text = 'Žluťoučký kůň'; + const html = `${text}`; + res.setHeader('content-type', 'text/html'); // no charset in HTTP header + res.end(iconv.encode(html, 'windows-1250')); +}); + let server: http.Server; let url: string; @@ -208,6 +223,36 @@ test('invalid content type defaults to octet-stream', async () => { ]); }); +test('decodes charset from http-equiv meta tag when absent in HTTP header', async () => { + const results: string[] = []; + + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: ({ body }) => { + results.push(body as string); + }, + }); + + await crawler.run([`${url}/meta-charset`]); + + expect(results[0]).toContain('Žluťoučký kůň'); +}); + +test('decodes charset from HTML5 meta charset attribute when absent in HTTP header', async () => { + const results: string[] = []; + + const crawler = new HttpCrawler({ + maxRequestRetries: 0, + requestHandler: ({ body }) => { + results.push(body as string); + }, + }); + + await crawler.run([`${url}/meta-charset-html5`]); + + expect(results[0]).toContain('Žluťoučký kůň'); +}); + test('handles cookies from redirects', async () => { const results: string[] = []; diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts index e1cc1c1cb5cc..ad9d0d215f0c 100644 --- a/test/shared/_helper.ts +++ b/test/shared/_helper.ts @@ -7,6 +7,7 @@ import bodyParser from 'body-parser'; import { entries } from 'crawlee'; import type { Application } from 'express'; import express from 'express'; +import iconv from 'iconv-lite'; export const startExpressAppPromise = async (app: Application, port: number) => { return new Promise((resolve) => { @@ -332,6 +333,13 @@ export async function runExampleComServer(): Promise<[Server, number]> { res.type('html').send('"<>"<>'); }); + special.get('/meta-charset', (_req, res) => { + const text = 'Žluťoučký kůň'; + const html = `${text}`; + res.setHeader('content-type', 'text/html'); + res.end(iconv.encode(html, 'windows-1250')); + }); + special.get('/set-cookie', (req, res) => { const cookieName = (req.query.name as string) || 'testCookie'; const cookieValue = (req.query.value as string) || 'testValue';