info providers: optionally fetch a part's data when its page is first viewed (to reduce cost/api calls)

assets/controllers/pages/provider_fetch_controller.js

@@ -0,0 +1,120 @@
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2026 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {Controller} from "@hotwired/stimulus";
+
+/**
+ * Shown on the info page of a part, which was not looked up at its info provider yet (and only if "fetch data when a
+ * part is viewed" is enabled for that provider). It asks the server to retrieve the data, shows that this is in
+ * progress, and reloads the page once the data is there.
+ */
+export default class extends Controller {
+    static targets = ["spinner", "done", "failed", "message"];
+
+    static values = {
+        url: String,
+        token: String,
+        messages: Object,
+    };
+
+    /** How often to ask again, while another request is retrieving the data of this part */
+    static MAX_ATTEMPTS = 60;
+    static RETRY_DELAY = 5000;
+
+    connect() {
+        //Turbo shows a cached copy of the page while it loads the real one, which will start the request itself
+        if (document.documentElement.hasAttribute('data-turbo-preview')) {
+            return;
+        }
+
+        this._attempts = 0;
+        this._waited = false;
+        this._fetch();
+    }
+
+    disconnect() {
+        clearTimeout(this._timeout);
+        this._timeout = null;
+        this._disconnected = true;
+    }
+
+    async _fetch() {
+        this._attempts++;
+
+        let status = 'error';
+        try {
+            const response = await fetch(this.urlValue, {
+                method: 'POST',
+                headers: {'X-Requested-With': 'XMLHttpRequest'},
+                body: new URLSearchParams({_token: this.tokenValue}),
+            });
+            if (response.ok) {
+                status = (await response.json()).status;
+            }
+        } catch (e) {
+            status = 'error';
+        }
+
+        //The user has left the page in the meantime
+        if (this._disconnected) {
+            return;
+        }
+
+        switch (status) {
+            case 'updated':
+                this._finish();
+                break;
+            case 'unchanged':
+                //If we had to wait for another request, that one has fetched the data, otherwise there was nothing to do
+                if (this._waited) {
+                    this._finish();
+                } else {
+                    this.element.remove();
+                }
+                break;
+            case 'busy':
+                if (this._attempts < this.constructor.MAX_ATTEMPTS) {
+                    this._waited = true;
+                    this._timeout = setTimeout(() => this._fetch(), this.constructor.RETRY_DELAY);
+                } else {
+                    this._fail('error');
+                }
+                break;
+            default:
+                this._fail(status === 'limit' ? 'limit' : 'error');
+        }
+    }
+
+    /** The data is there: say so, and show the page with the new data */
+    _finish() {
+        this.spinnerTarget.classList.add('d-none');
+        this.doneTarget.classList.remove('d-none');
+        this.element.classList.replace('alert-info', 'alert-success');
+        this.messageTarget.textContent = this.messagesValue.updated;
+
+        window.Turbo.visit(window.location.href, {action: 'replace'});
+    }
+
+    _fail(reason) {
+        this.spinnerTarget.classList.add('d-none');
+        this.failedTarget.classList.remove('d-none');
+        this.element.classList.replace('alert-info', 'alert-warning');
+        this.messageTarget.textContent = this.messagesValue[reason];
+    }
+}

docs/usage/information_provider_system.md

@@ -386,6 +386,12 @@ Once you have the API key, you can configure the Canopy provider in Part-DB usin
 
 * `PROVIDER_CANOPY_API_KEY`: The API key you got from Canopy (mandatory)
 
+As Canopy bills per request, retrieving data for all Amazon parts of a large inventory can be expensive. With the
+*Fetch data when a part is viewed* option in the provider settings, Part-DB only asks Canopy about an Amazon part, when
+somebody opens its info page for the first time (see [Fetching data when a part is viewed](#fetching-data-when-a-part-is-viewed)).
+This applies to parts which have an orderdetail linking to a product page of the configured Amazon marketplace, and the
+number of requests caused this way is capped by *Max. requests per day when viewing parts* of the provider settings.
+
 ### SparkFun
 
 The SparkFun provider retrieves product information from [sparkfun.com](https://www.sparkfun.com/). You can search by
@@ -509,6 +515,41 @@ the actual API requests and return the information to Part-DB.
 See the existing providers for examples.
 If you created a new provider, feel free to create a pull request to add it to the Part-DB core.
 
+## Fetching data when a part is viewed
+
+Parts which were created by hand or by an import have no info provider data. Instead of looking all of them up at once
+(which costs money at providers billing per request, and is a lot of traffic for the website of a small store), Part-DB
+can retrieve the data of such a part when somebody opens its info page for the first time: the page is shown as usual,
+says that data is being fetched from the provider, and reloads with the new data once it is there.
+
+This is off by default. Select the providers which should do this under *Fetch data when a part is viewed* in the
+general info provider settings, or list their keys in the `PROVIDER_FETCH_ON_VIEW` environment variable (comma separated,
+e.g. `PROVIDER_FETCH_ON_VIEW=lcsc,pollin`). Only active providers are used. The Canopy provider additionally has its
+own switch for this in its settings.
+
+A part is only looked up if it has no info provider reference yet and one of its orderdetails leads to one of the
+selected providers, checked in this order:
+
+1. The product URL of the orderdetail is a product page of the provider (this works for the providers which can
+   create a part from an URL, and for Amazon URLs with Canopy). The URL identifies the product.
+2. The supplier of the orderdetail has the same name as the provider (ignoring case, spaces and punctuation) and the
+   orderdetail has a supplier part number. The provider is searched for that number, and the result is only used if
+   its ID, manufacturer part number or order number is exactly the supplier part number (ignoring case and a vendor
+   prefix, so `4062` matches `ADA4062` and `13975` matches `DEV-13975`). Part-DB never picks a merely similar product.
+
+Parts with neither are left alone, and checking this does not contact the provider.
+
+As nobody reviews the result, only missing data is filled in (description, notes, manufacturer, manufacturer part
+number, pictures and datasheets, parameters, and prices if the orderdetail has none); existing data is never changed.
+Afterwards the part carries the info provider reference, so it is not looked up again, and it can be updated from the
+provider with the normal tools later.
+
+The number of parts looked up this way is capped per provider by *Max. lookups per day when viewing parts*
+(`PROVIDER_FETCH_ON_VIEW_DAILY_LIMIT`, default 100, 0 for no limit); when it is reached, the page says so and the part
+is tried again on a later view. If the provider fails, refuses the request, or does not know the supplier part number,
+the page says that the data could not be fetched and the part is not tried again for 24 hours. Everybody who is allowed
+to view a part triggers the lookup.
+
 ## Result caching
 
 To reduce the number of API calls against the providers, the results are cached:

src/Controller/PartController.php

@@ -42,6 +42,7 @@
 use App\Services\Attachments\PartPreviewGenerator;
 use App\Services\EntityMergers\Mergers\PartMerger;
 use App\Services\InfoProviderSystem\PartInfoRetriever;
+use App\Services\InfoProviderSystem\ProviderOnViewFetcher;
 use App\Services\InfoProviderSystem\Providers\AIDocumentProvider;
 use App\Services\InfoProviderSystem\UploadedDocumentAttachmentHelper;
 use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
@@ -101,6 +102,7 @@ public function show(
         DataTableFactory $dataTable,
         ParameterExtractor $parameterExtractor,
         PartLotWithdrawAddHelper $withdrawAddHelper,
+        ProviderOnViewFetcher $onViewFetcher,
         ?string $timestamp = null
     ): Response {
         $this->denyAccessUnlessGranted('read', $part);
@@ -168,10 +170,46 @@ public function show(
                 'highlightLotId' => $request->query->getInt('highlightLot', 0),
                 'add_lot_form' => $addLotForm,
                 'move_new_lot_form' => $moveNewLotForm,
+                //The page itself triggers the (slow) request to the info provider afterwards, so rendering is not delayed
+                'fetch_on_view_provider' => $timeTravel_timestamp === null
+                    ? $onViewFetcher->getPendingMatch($part)?->getProviderName() : null,
             ]
         );
     }
 
+    /**
+     * Fills a part with data from the info provider which recognizes it. Called by the part info page, if "fetch
+     * data when a part is viewed" is enabled for that provider and the part was not looked up yet.
+     * Being allowed to view the part is enough to trigger this: the administrator opted in to that with the setting,
+     * and only missing data is added. The number of lookups is bounded by the daily limit of the setting.
+     */
+    #[Route(path: '/{id}/fetch_on_view', name: 'part_fetch_on_view', requirements: ['id' => '\d+'], methods: ['POST'])]
+    public function fetchOnView(Part $part, Request $request, ProviderOnViewFetcher $onViewFetcher): Response
+    {
+        $this->denyAccessUnlessGranted('read', $part);
+
+        if (!$this->isCsrfTokenValid('fetch_on_view' . $part->getID(), $request->request->get('_token'))) {
+            throw $this->createAccessDeniedException('Invalid CSRF token');
+        }
+
+        $provider_name = $onViewFetcher->getPendingMatch($part)?->getProviderName();
+
+        //A fetch can take a while (some providers pause between their requests to a store). The session is locked
+        //as long as it is open, so close it, to not block all other pages of this user in the meantime.
+        if ($request->hasSession() && $request->getSession()->isStarted()) {
+            $request->getSession()->save();
+        }
+
+        $status = $onViewFetcher->fetch($part);
+
+        //The page reloads itself after a successful fetch, so tell the user why the part looks different now
+        if ($status === ProviderOnViewFetcher::STATUS_UPDATED) {
+            $this->addFlash('success', t('part.info.provider_fetch.flash.updated', ['%provider%' => (string) $provider_name]));
+        }
+
+        return $this->json(['status' => $status]);
+    }
+
     #[Route(path: '/{id}/add_lot', name: 'part_lot_add', methods: ['POST'])]
     public function addLot(Part $part, Request $request, EntityManagerInterface $em): Response
     {

src/Services/InfoProviderSystem/OnViewMatch.php

@@ -0,0 +1,59 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2026 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\Services\InfoProviderSystem;
+
+use App\Entity\PriceInformations\Orderdetail;
+use App\Services\InfoProviderSystem\Providers\InfoProviderInterface;
+
+/**
+ * Describes how a part without info provider data was recognized as a product of an info provider, by one of its
+ * orderdetails (see ProviderOnViewMatcher).
+ */
+final readonly class OnViewMatch
+{
+    /**
+     * @param  InfoProviderInterface  $provider The provider which can supply the data of the part
+     * @param  Orderdetail  $orderdetail The orderdetail of the part, which led to the provider
+     * @param  string|null  $providerId The ID of the product at the provider, if the product URL of the orderdetail
+     * gave it away. Null if the product has to be looked up by the supplier part number first.
+     * @param  string|null  $supplierPartNr The supplier part number to look the product up with, if $providerId is null
+     */
+    public function __construct(
+        public InfoProviderInterface $provider,
+        public Orderdetail $orderdetail,
+        public ?string $providerId = null,
+        public ?string $supplierPartNr = null,
+    ) {
+    }
+
+    public function getProviderKey(): string
+    {
+        return $this->provider->getProviderInfo()->key;
+    }
+
+    public function getProviderName(): string
+    {
+        return $this->provider->getProviderInfo()->name;
+    }
+}