feat: align EnqueueLinksOptions with crawlee-python

docs/deployment/apify_platform_init_exit.ts

@@ -13,7 +13,7 @@ const crawler = new CheerioCrawler({
 
         // Add URLs that match the provided pattern.
         await enqueueLinks({
-            globs: ['https://www.iana.org/*'],
+            include: ['https://www.iana.org/*'],
         });
 
         // Save extracted data to dataset.

docs/deployment/apify_platform_main.ts

@@ -12,7 +12,7 @@ await Actor.main(async () => {
 
             // Add URLs that match the provided pattern.
             await enqueueLinks({
-                globs: ['https://www.iana.org/*'],
+                include: ['https://www.iana.org/*'],
             });
 
             // Save extracted data to dataset.

docs/examples/crawl_some_links.mdx

@@ -7,7 +7,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 import ApiLink from '@site/src/components/ApiLink';
 import CrawlSource from '!!raw-loader!roa-loader!./crawl_some_links.ts';
 
-This <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink> example uses the <ApiLink to="core/interface/EnqueueLinksOptions#globs">`globs`</ApiLink> property in the <ApiLink to="cheerio-crawler/interface/CheerioCrawlingContext#enqueueLinks">`enqueueLinks()`</ApiLink> method to only add links to the <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> queue if they match the specified pattern.
+This <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink> example uses the <ApiLink to="core/interface/EnqueueLinksOptions#include">`include`</ApiLink> property in the <ApiLink to="cheerio-crawler/interface/CheerioCrawlingContext#enqueueLinks">`enqueueLinks()`</ApiLink> method to only add links to the <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> queue if they match the specified pattern.
 
 <RunnableCodeBlock className="language-js" type="cheerio">
 	{CrawlSource}

docs/examples/crawl_some_links.ts

@@ -9,7 +9,7 @@ const crawler = new CheerioCrawler({
         log.info(request.url);
         // Add some links from page to the crawler's RequestQueue
         await enqueueLinks({
-            globs: ['http?(s)://crawlee.dev/*/*'],
+            include: ['http?(s)://crawlee.dev/*/*'],
         });
     },
 });

docs/examples/puppeteer_recursive_crawl.ts

@@ -6,7 +6,7 @@ const crawler = new PuppeteerCrawler({
         log.info(`Title of ${request.url}: ${title}`);
 
         await enqueueLinks({
-            globs: ['http?(s)://www.iana.org/**'],
+            include: ['http?(s)://www.iana.org/**'],
         });
     },
     maxRequestsPerCrawl: 10,

docs/guides/request_loaders_sitemap_basic.ts

@@ -5,7 +5,7 @@ import { SitemapRequestLoader } from 'crawlee';
 const sitemapRequestLoader = await SitemapRequestLoader.open({
     sitemapUrls: ['https://crawlee.dev/sitemap.xml'],
     // Optionally filter the URLs read from the sitemap:
-    // globs: ['https://crawlee.dev/docs/**'],
+    // include: ['https://crawlee.dev/docs/**'],
 });
 
 for await (const request of sitemapRequestLoader) {

docs/introduction/03-adding-urls.mdx

@@ -130,7 +130,7 @@ await enqueueLinks({
 
 ### Filter URLs with patterns
 
-For even more control, you can use `globs`, `regexps` and `pseudoUrls` to filter the URLs. Each of those arguments is always an `Array`, but the contents can take on many forms. <ApiLink to="core/interface/EnqueueLinksOptions">See the reference</ApiLink> for more information about them as well as other options.
+For even more control, you can use `include` and `exclude` to filter the URLs. Each accepts an `Array` of glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects. <ApiLink to="core/interface/EnqueueLinksOptions">See the reference</ApiLink> for more information about them as well as other options.
 
 :::caution Defaults override
 
@@ -140,17 +140,17 @@ If you provide one of those options, the default `same-hostname` strategy will *
 
 ```ts
 await enqueueLinks({
-    globs: ['http?(s)://apify.com/*/*'],
+    include: ['http?(s)://apify.com/*/*'],
 });
 ```
 
 ### Transform requests
 
-To have absolute control, we have the <ApiLink to="core/interface/EnqueueLinksOptions/#transformRequestFunction">`transformRequestFunction`</ApiLink>. Just before a new <ApiLink to="core/class/Request">`Request`</ApiLink> is constructed and enqueued to the <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>, this function can be used to skip it or modify its contents such as `userData`, `payload` or, most importantly, `uniqueKey`. This is useful when you need to enqueue multiple requests to the queue, and these requests share the same URL, but differ in methods or payloads. Another use case is to dynamically update or create the `userData`.
+To have absolute control, we have the <ApiLink to="core/interface/EnqueueLinksOptions/#transformRequestFunction">`transformRequestFunction`</ApiLink>. After request options are filtered by `include`/`exclude` patterns, this function can be used to skip them or modify their contents such as `userData`, `payload` or, most importantly, `uniqueKey`. This is useful when you need to enqueue multiple requests to the queue, and these requests share the same URL, but differ in methods or payloads. Another use case is to dynamically update or create the `userData`.
 
 ```ts
 await enqueueLinks({
-    globs: ['http?(s)://apify.com/*/*'],
+    include: ['http?(s)://apify.com/*/*'],
     transformRequestFunction(req) {
         // ignore all links ending with `.pdf`
         if (req.url.endsWith('.pdf')) return false;

docs/upgrading/upgrading_v3.md

@@ -188,14 +188,13 @@ One common helper that received more attention is the `enqueueLinks`. As mention
 
 This means we can even call `enqueueLinks()` without any parameters. By default, it will go through all the links found on current page and filter only those targeting the same subdomain.
 
-Moreover, we can specify patterns the URL should match via globs:
+Moreover, we can specify patterns the URL should match via `include`:
 
 ```ts
 const crawler = new PlaywrightCrawler({
     async requestHandler({ enqueueLinks }) {
         await enqueueLinks({
-            globs: ['https://crawlee.dev/*/*'],
-            // we can also use `regexps` and `pseudoUrls` keys here
+            include: ['https://crawlee.dev/*/*'],
         });
     },
 });
@@ -231,7 +230,7 @@ Labeling requests used to work via the `Request.userData` object. With Crawlee,
 async requestHandler({ request, enqueueLinks }) {
     if (request.label !== 'DETAIL') {
         await enqueueLinks({
-            globs: ['...'],
+            include: ['...'],
             label: 'DETAIL',
         });
     }

docs/upgrading/upgrading_v4.md

@@ -897,14 +897,37 @@ await enqueueLinks({ urls, requestQueue });
 await enqueueLinks({ urls, requestManager });
 ```
 
+### `globs`, `regexps`, and `pseudoUrls` replaced by `include`
+
+To align with the Crawlee for Python API, the separate `globs`, `regexps`, and `pseudoUrls` URL-filtering options of `enqueueLinks()`, the click-elements enqueue helpers, and `SitemapRequestLoader` have been collapsed into a single `include` option (mirroring the already-unified `exclude` option). Each entry of `include`/`exclude` can be a glob string, a `RegExp`, or a `{ glob }` / `{ regexp }` object.
+
+The `PseudoUrl` class is no longer exported and the `@apify/pseudo_url` dependency has been dropped. Rewrite any pseudo-URL patterns as globs or regular expressions.
+
+Per-pattern request options (`label`, `userData`, `method`, `payload`, `headers` set directly on a pattern object) are no longer supported. Use the top-level `label` / `userData` options, or `transformRequestFunction`, to set request options for the enqueued requests.
+
+**Before:**
+```typescript
+await enqueueLinks({
+    globs: ['https://crawlee.dev/docs/**'],
+    regexps: [/\/blog\//],
+    pseudoUrls: ['https://crawlee.dev/[.*]'],
+});
+```
+
+**After:**
+```typescript
+await enqueueLinks({
+    include: ['https://crawlee.dev/docs/**', /\/blog\//, 'https://crawlee.dev/**'],
+});
+```
+
 ## `transformRequestFunction` precedence in `enqueueLinks`
 
-The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`globs`, `regexps`, `pseudoUrls`) instead of before. This means it has the highest priority and can overwrite any request options set by patterns or the global `label` option.
+The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`include`, `exclude`) instead of before. This means it has the highest priority and can overwrite any request options set by the global `label` / `userData` options.
 
 The priority order is now (lowest to highest):
 1. Global `label` / `userData` options
-2. Pattern-specific options from `globs`, `regexps`, or `pseudoUrls` objects
-3. `transformRequestFunction`
+2. `transformRequestFunction`
 
 The `transformRequestFunction` callback receives a `RequestOptions` object and can return either:
 - The modified `RequestOptions` object

packages/core/package.json

@@ -50,7 +50,6 @@
         "@apify/consts": "^2.41.0",
         "@apify/datastructures": "^2.0.3",
         "@apify/log": "^2.5.18",
-        "@apify/pseudo_url": "^2.0.59",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
         "@crawlee/fs-storage": "workspace:*",

packages/core/src/crawlers/crawler_commons.ts

@@ -57,8 +57,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -68,7 +67,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });
@@ -115,8 +114,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
      * currently used by the crawler.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
-     * and override settings of the enqueued {@apilink Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * Check out the [Crawl a website with relative links](https://crawlee.dev/js/docs/examples/crawl-relative-links) example
      * for more details regarding its usage.
@@ -126,7 +124,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * ```ts
      * async requestHandler({ enqueueLinks }) {
      *     await enqueueLinks({
-     *       globs: [
+     *       include: [
      *           'https://www.example.com/handbags/*',
      *       ],
      *     });