fix(core): respect Retry-After header on 429 without retiring session

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -41,6 +41,7 @@ import {
     mergeCookies,
     NonRetryableError,
     purgeDefaultStorages,
+    RateLimitError,
     RequestListAdapter,
     RequestManagerTandem,
     RequestProvider,
@@ -1834,6 +1835,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
             if (error instanceof SessionError) {
                 await this._rotateSession(crawlingContext);
+            } else if (error instanceof RateLimitError) {
+                const domain = getDomain(request.url);
+                // We default to 1 minute if retryAfterMs is not provided
+                const cooldown = error.retryAfterMs ?? 60000;
+                if (domain) {
+                    this.domainAccessedTime.set(domain, Date.now() + cooldown - this.sameDomainDelayMillis);
+                }
             }
 
             if (!request.noRetry) {
@@ -1932,7 +1940,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
         }
 
         // User requested retry (we ignore retry count here as its explicitly told by the user to retry)
-        if (error instanceof RetryRequestError) {
+        // Rate limit errors are also explicitly retried (with cooldown).
+        if (error instanceof RetryRequestError || error instanceof RateLimitError) {
             return true;
         }
 

packages/core/src/errors.ts

@@ -35,3 +35,15 @@ export class SessionError extends RetryRequestError {
         super(`Detected a session error, rotating session... ${message ? `\n${message}` : ''}`);
     }
 }
+
+/**
+ * Errors of `RateLimitError` type will trigger a cooldown before the request is retried.
+ */
+export class RateLimitError extends Error {
+    constructor(
+        message?: string,
+        public readonly retryAfterMs?: number,
+    ) {
+        super(message ?? 'Rate limit exceeded');
+    }
+}

packages/core/src/session_pool/consts.ts

@@ -1,3 +1,3 @@
-export const BLOCKED_STATUS_CODES = [401, 403, 429];
+export const BLOCKED_STATUS_CODES = [401, 403];
 export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;

packages/core/src/session_pool/session.ts

@@ -296,10 +296,10 @@ export class Session {
     }
 
     /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally the default status
-     * codes can be extended in the second parameter.
+     * Retires session based on status code.
+     * With certain status codes: `401` or `403` we can be certain
+     * that the IP is blocked.
+     *
      * @param statusCode HTTP status code.
      * @returns Whether the session was retired.
      */

packages/core/src/session_pool/session_pool.ts

@@ -56,7 +56,7 @@ export interface SessionPoolOptions {
     /**
      * Specifies which response status codes are considered as blocked.
      * Session connected to such request will be marked as retired.
-     * @default [401, 403, 429]
+     * @default [401, 403]
      */
     blockedStatusCodes?: number[];
 

packages/http-crawler/src/internals/http-crawler.ts

@@ -26,6 +26,8 @@ import {
     processHttpRequestOptions,
     RequestState,
     Router,
+    NonRetryableError,
+    RateLimitError,
     SessionError,
     validators,
 } from '@crawlee/basic';
@@ -517,6 +519,23 @@ export class HttpCrawler<
             await this._handleNavigation(crawlingContext);
             tryCancel();
 
+            if (crawlingContext.response?.statusCode === 429) {
+                const retryAfterHeader = crawlingContext.response.headers['retry-after'];
+                let retryAfterMs: number | undefined = undefined;
+                if (retryAfterHeader) {
+                    const retryAfterStr = String(retryAfterHeader);
+                    if (/^\d+$/.test(retryAfterStr)) {
+                        retryAfterMs = parseInt(retryAfterStr, 10) * 1000;
+                    } else {
+                        const date = Date.parse(retryAfterStr);
+                        if (!Number.isNaN(date)) {
+                            retryAfterMs = Math.max(0, date - Date.now());
+                        }
+                    }
+                }
+                throw new RateLimitError(`Rate limit exceeded (HTTP 429)`, retryAfterMs);
+            }
+
             const parsed = await this._parseResponse(request, crawlingContext.response!, crawlingContext);
             const response = parsed.response!;
             const contentType = parsed.contentType!;

test/core/crawlers/http_crawler.test.ts

@@ -61,6 +61,21 @@ router.set('/403-with-octet-stream', (req, res) => {
     res.end();
 });
 
+let rateLimitHit = 0;
+router.set('/429', (req, res) => {
+    if (rateLimitHit === 0) {
+        rateLimitHit++;
+        res.setHeader('content-type', 'text/html');
+        res.setHeader('retry-after', '1');
+        res.statusCode = 429;
+        res.end();
+    } else {
+        res.setHeader('content-type', 'text/html');
+        res.statusCode = 200;
+        res.end('ok');
+    }
+});
+
 let server: http.Server;
 let url: string;
 
@@ -89,6 +104,7 @@ afterAll(async () => {
 const localStorageEmulator = new MemoryStorageEmulator();
 
 beforeEach(async () => {
+    rateLimitHit = 0;
     await localStorageEmulator.init();
 });
 
@@ -477,4 +493,21 @@ describe.each(
         expect(results[0].includes('Schmexample Domain')).toBeTruthy();
         expect(results[1].includes('Hello')).toBeTruthy();
     });
+    test('should respect 429 RateLimitError and retry', async () => {
+        const results: string[] = [];
+        const crawler = new HttpCrawler({
+            httpClient,
+            maxRequestRetries: 1,
+            requestHandler: async ({ body }) => {
+                results.push(body.toString());
+            },
+        });
+
+        const start = Date.now();
+        await crawler.run([`${url}/429`]);
+        const duration = Date.now() - start;
+
+        expect(results).toStrictEqual(['ok']);
+        expect(duration).toBeGreaterThanOrEqual(800); // Because of retry-after: 1, allow some tolerance
+    });
 });