diff --git a/assets/controllers/pages/provider_fetch_controller.js b/assets/controllers/pages/provider_fetch_controller.js new file mode 100644 index 000000000..bff739a96 --- /dev/null +++ b/assets/controllers/pages/provider_fetch_controller.js @@ -0,0 +1,120 @@ +/* + * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony). + * + * Copyright (C) 2019 - 2026 Jan Böhmer (https://github.com/jbtronics) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import {Controller} from "@hotwired/stimulus"; + +/** + * Shown on the info page of a part, which was not looked up at its info provider yet (and only if "fetch data when a + * part is viewed" is enabled for that provider). It asks the server to retrieve the data, shows that this is in + * progress, and reloads the page once the data is there. + */ +export default class extends Controller { + static targets = ["spinner", "done", "failed", "message"]; + + static values = { + url: String, + token: String, + messages: Object, + }; + + /** How often to ask again, while another request is retrieving the data of this part */ + static MAX_ATTEMPTS = 60; + static RETRY_DELAY = 5000; + + connect() { + //Turbo shows a cached copy of the page while it loads the real one, which will start the request itself + if (document.documentElement.hasAttribute('data-turbo-preview')) { + return; + } + + this._attempts = 0; + this._waited = false; + this._fetch(); + } + + disconnect() { + clearTimeout(this._timeout); + this._timeout = null; + this._disconnected = true; + } + + async _fetch() { + this._attempts++; + + let status = 'error'; + try { + const response = await fetch(this.urlValue, { + method: 'POST', + headers: {'X-Requested-With': 'XMLHttpRequest'}, + body: new URLSearchParams({_token: this.tokenValue}), + }); + if (response.ok) { + status = (await response.json()).status; + } + } catch (e) { + status = 'error'; + } + + //The user has left the page in the meantime + if (this._disconnected) { + return; + } + + switch (status) { + case 'updated': + this._finish(); + break; + case 'unchanged': + //If we had to wait for another request, that one has fetched the data, otherwise there was nothing to do + if (this._waited) { + this._finish(); + } else { + this.element.remove(); + } + break; + case 'busy': + if (this._attempts < this.constructor.MAX_ATTEMPTS) { + this._waited = true; + this._timeout = setTimeout(() => this._fetch(), this.constructor.RETRY_DELAY); + } else { + this._fail('error'); + } + break; + default: + this._fail(status === 'limit' ? 'limit' : 'error'); + } + } + + /** The data is there: say so, and show the page with the new data */ + _finish() { + this.spinnerTarget.classList.add('d-none'); + this.doneTarget.classList.remove('d-none'); + this.element.classList.replace('alert-info', 'alert-success'); + this.messageTarget.textContent = this.messagesValue.updated; + + window.Turbo.visit(window.location.href, {action: 'replace'}); + } + + _fail(reason) { + this.spinnerTarget.classList.add('d-none'); + this.failedTarget.classList.remove('d-none'); + this.element.classList.replace('alert-info', 'alert-warning'); + this.messageTarget.textContent = this.messagesValue[reason]; + } +} diff --git a/docs/usage/information_provider_system.md b/docs/usage/information_provider_system.md index d8a41e364..0a6b9cd61 100644 --- a/docs/usage/information_provider_system.md +++ b/docs/usage/information_provider_system.md @@ -386,6 +386,12 @@ Once you have the API key, you can configure the Canopy provider in Part-DB usin * `PROVIDER_CANOPY_API_KEY`: The API key you got from Canopy (mandatory) +As Canopy bills per request, retrieving data for all Amazon parts of a large inventory can be expensive. With the +*Fetch data when a part is viewed* option in the provider settings, Part-DB only asks Canopy about an Amazon part, when +somebody opens its info page for the first time (see [Fetching data when a part is viewed](#fetching-data-when-a-part-is-viewed)). +This applies to parts which have an orderdetail linking to a product page of the configured Amazon marketplace, and the +number of requests caused this way is capped by *Max. requests per day when viewing parts* of the provider settings. + ### SparkFun The SparkFun provider retrieves product information from [sparkfun.com](https://www.sparkfun.com/). You can search by @@ -509,6 +515,41 @@ the actual API requests and return the information to Part-DB. See the existing providers for examples. If you created a new provider, feel free to create a pull request to add it to the Part-DB core. +## Fetching data when a part is viewed + +Parts which were created by hand or by an import have no info provider data. Instead of looking all of them up at once +(which costs money at providers billing per request, and is a lot of traffic for the website of a small store), Part-DB +can retrieve the data of such a part when somebody opens its info page for the first time: the page is shown as usual, +says that data is being fetched from the provider, and reloads with the new data once it is there. + +This is off by default. Select the providers which should do this under *Fetch data when a part is viewed* in the +general info provider settings, or list their keys in the `PROVIDER_FETCH_ON_VIEW` environment variable (comma separated, +e.g. `PROVIDER_FETCH_ON_VIEW=lcsc,pollin`). Only active providers are used. The Canopy provider additionally has its +own switch for this in its settings. + +A part is only looked up if it has no info provider reference yet and one of its orderdetails leads to one of the +selected providers, checked in this order: + +1. The product URL of the orderdetail is a product page of the provider (this works for the providers which can + create a part from an URL, and for Amazon URLs with Canopy). The URL identifies the product. +2. The supplier of the orderdetail has the same name as the provider (ignoring case, spaces and punctuation) and the + orderdetail has a supplier part number. The provider is searched for that number, and the result is only used if + its ID, manufacturer part number or order number is exactly the supplier part number (ignoring case and a vendor + prefix, so `4062` matches `ADA4062` and `13975` matches `DEV-13975`). Part-DB never picks a merely similar product. + +Parts with neither are left alone, and checking this does not contact the provider. + +As nobody reviews the result, only missing data is filled in (description, notes, manufacturer, manufacturer part +number, pictures and datasheets, parameters, and prices if the orderdetail has none); existing data is never changed. +Afterwards the part carries the info provider reference, so it is not looked up again, and it can be updated from the +provider with the normal tools later. + +The number of parts looked up this way is capped per provider by *Max. lookups per day when viewing parts* +(`PROVIDER_FETCH_ON_VIEW_DAILY_LIMIT`, default 100, 0 for no limit); when it is reached, the page says so and the part +is tried again on a later view. If the provider fails, refuses the request, or does not know the supplier part number, +the page says that the data could not be fetched and the part is not tried again for 24 hours. Everybody who is allowed +to view a part triggers the lookup. + ## Result caching To reduce the number of API calls against the providers, the results are cached: diff --git a/src/Controller/PartController.php b/src/Controller/PartController.php index b26d752b2..ef092516f 100644 --- a/src/Controller/PartController.php +++ b/src/Controller/PartController.php @@ -42,6 +42,7 @@ use App\Services\Attachments\PartPreviewGenerator; use App\Services\EntityMergers\Mergers\PartMerger; use App\Services\InfoProviderSystem\PartInfoRetriever; +use App\Services\InfoProviderSystem\ProviderOnViewFetcher; use App\Services\InfoProviderSystem\Providers\AIDocumentProvider; use App\Services\InfoProviderSystem\UploadedDocumentAttachmentHelper; use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; @@ -101,6 +102,7 @@ public function show( DataTableFactory $dataTable, ParameterExtractor $parameterExtractor, PartLotWithdrawAddHelper $withdrawAddHelper, + ProviderOnViewFetcher $onViewFetcher, ?string $timestamp = null ): Response { $this->denyAccessUnlessGranted('read', $part); @@ -168,10 +170,46 @@ public function show( 'highlightLotId' => $request->query->getInt('highlightLot', 0), 'add_lot_form' => $addLotForm, 'move_new_lot_form' => $moveNewLotForm, + //The page itself triggers the (slow) request to the info provider afterwards, so rendering is not delayed + 'fetch_on_view_provider' => $timeTravel_timestamp === null + ? $onViewFetcher->getPendingMatch($part)?->getProviderName() : null, ] ); } + /** + * Fills a part with data from the info provider which recognizes it. Called by the part info page, if "fetch + * data when a part is viewed" is enabled for that provider and the part was not looked up yet. + * Being allowed to view the part is enough to trigger this: the administrator opted in to that with the setting, + * and only missing data is added. The number of lookups is bounded by the daily limit of the setting. + */ + #[Route(path: '/{id}/fetch_on_view', name: 'part_fetch_on_view', requirements: ['id' => '\d+'], methods: ['POST'])] + public function fetchOnView(Part $part, Request $request, ProviderOnViewFetcher $onViewFetcher): Response + { + $this->denyAccessUnlessGranted('read', $part); + + if (!$this->isCsrfTokenValid('fetch_on_view' . $part->getID(), $request->request->get('_token'))) { + throw $this->createAccessDeniedException('Invalid CSRF token'); + } + + $provider_name = $onViewFetcher->getPendingMatch($part)?->getProviderName(); + + //A fetch can take a while (some providers pause between their requests to a store). The session is locked + //as long as it is open, so close it, to not block all other pages of this user in the meantime. + if ($request->hasSession() && $request->getSession()->isStarted()) { + $request->getSession()->save(); + } + + $status = $onViewFetcher->fetch($part); + + //The page reloads itself after a successful fetch, so tell the user why the part looks different now + if ($status === ProviderOnViewFetcher::STATUS_UPDATED) { + $this->addFlash('success', t('part.info.provider_fetch.flash.updated', ['%provider%' => (string) $provider_name])); + } + + return $this->json(['status' => $status]); + } + #[Route(path: '/{id}/add_lot', name: 'part_lot_add', methods: ['POST'])] public function addLot(Part $part, Request $request, EntityManagerInterface $em): Response { diff --git a/src/Services/InfoProviderSystem/OnViewMatch.php b/src/Services/InfoProviderSystem/OnViewMatch.php new file mode 100644 index 000000000..dac676237 --- /dev/null +++ b/src/Services/InfoProviderSystem/OnViewMatch.php @@ -0,0 +1,59 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem; + +use App\Entity\PriceInformations\Orderdetail; +use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; + +/** + * Describes how a part without info provider data was recognized as a product of an info provider, by one of its + * orderdetails (see ProviderOnViewMatcher). + */ +final readonly class OnViewMatch +{ + /** + * @param InfoProviderInterface $provider The provider which can supply the data of the part + * @param Orderdetail $orderdetail The orderdetail of the part, which led to the provider + * @param string|null $providerId The ID of the product at the provider, if the product URL of the orderdetail + * gave it away. Null if the product has to be looked up by the supplier part number first. + * @param string|null $supplierPartNr The supplier part number to look the product up with, if $providerId is null + */ + public function __construct( + public InfoProviderInterface $provider, + public Orderdetail $orderdetail, + public ?string $providerId = null, + public ?string $supplierPartNr = null, + ) { + } + + public function getProviderKey(): string + { + return $this->provider->getProviderInfo()->key; + } + + public function getProviderName(): string + { + return $this->provider->getProviderInfo()->name; + } +} diff --git a/src/Services/InfoProviderSystem/ProviderOnViewFetcher.php b/src/Services/InfoProviderSystem/ProviderOnViewFetcher.php new file mode 100644 index 000000000..fb0a29b6d --- /dev/null +++ b/src/Services/InfoProviderSystem/ProviderOnViewFetcher.php @@ -0,0 +1,377 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem; + +use App\Entity\Attachments\Attachment; +use App\Entity\Parameters\AbstractParameter; +use App\Entity\Parts\InfoProviderReference; +use App\Entity\Parts\ManufacturingStatus; +use App\Entity\Parts\Part; +use App\Entity\PriceInformations\Orderdetail; +use App\Services\LogSystem\EventCommentHelper; +use Doctrine\ORM\EntityManagerInterface; +use Psr\Cache\CacheItemPoolInterface; +use Psr\Log\LoggerInterface; +use Symfony\Component\RateLimiter\RateLimiterFactory; +use Symfony\Component\RateLimiter\Storage\CacheStorage; + +/** + * Fills a part with data from an info provider the first time its info page is opened. + * + * Looking up every part of a large inventory up front is expensive (Canopy bills per request) or impolite (many + * requests to the website of a store), while most of these parts are never looked at again. For the providers this + * is enabled for (see ProviderOnViewMatcher), a part is only looked up when somebody actually opens its page - and + * only once: afterwards the part carries a provider reference, which marks it as done. + * + * Only missing data is filled in. Nothing the part already has is overwritten or extended, as nobody reviews + * the result (in contrast to the "update from info provider" form). + */ +final class ProviderOnViewFetcher +{ + public const STATUS_UPDATED = 'updated'; + /** Nothing to do (anymore), e.g. because a parallel request already fetched the data */ + public const STATUS_UNCHANGED = 'unchanged'; + /** Another request is fetching the data for this part right now */ + public const STATUS_BUSY = 'busy'; + /** The daily request limit is used up */ + public const STATUS_LIMIT = 'limit'; + public const STATUS_ERROR = 'error'; + + /** + * @var int Seconds after which a fetch which never finished (e.g. a killed request) does not block new ones anymore. + * A fetch can take a while, as some providers wait several seconds between their requests to a store. + */ + private const LOCK_TTL = 300; + /** @var int Seconds before a part whose fetch failed is tried again, so a broken product does not cost a request per page view */ + private const FAILURE_TTL = 3600 * 24; + + public function __construct( + private readonly ProviderOnViewMatcher $matcher, + private readonly PartInfoRetriever $infoRetriever, + private readonly EntityManagerInterface $em, + private readonly EventCommentHelper $commentHelper, + private readonly CacheItemPoolInterface $partInfoCache, + private readonly LoggerInterface $logger, + ) { + } + + /** + * Returns how the data of the given part would be fetched, or null if there is nothing to fetch: the part has + * info provider data already, no enabled provider recognizes it, or fetching it failed recently. + * This check is cheap and does not contact a provider, so it can be done on every page view. + */ + public function getPendingMatch(Part $part): ?OnViewMatch + { + if ($part->getID() === null) { + return null; + } + + foreach ($this->matcher->findMatches($part) as $match) { + if (!$this->partInfoCache->hasItem($this->failureKey($part, $match))) { + return $match; + } + } + + return null; + } + + /** + * Checks whether opening the page of this part should trigger a request to an info provider. + */ + public function isFetchNeeded(Part $part): bool + { + return $this->getPendingMatch($part) !== null; + } + + /** + * Retrieves the data for the given part from the provider which recognizes it, and fills in what the part is + * missing. Whatever goes wrong at the provider ends up as STATUS_ERROR, this does not throw. + * @return string One of the STATUS_* constants + */ + public function fetch(Part $part): string + { + $match = $this->getPendingMatch($part); + if ($match === null) { + return self::STATUS_UNCHANGED; + } + + $lock_key = 'on_view_lock_'.$part->getID(); + $lock = $this->partInfoCache->getItem($lock_key); + if ($lock->isHit()) { + return self::STATUS_BUSY; + } + + if (!$this->consumeDailyLimit($match->getProviderKey())) { + return self::STATUS_LIMIT; + } + + $lock->set(true); + $lock->expiresAfter(self::LOCK_TTL); + $this->partInfoCache->save($lock); + + try { + $dto = $this->infoRetriever->getDetails($match->getProviderKey(), $this->resolveProviderId($match)); + $this->fillPart($part, $this->infoRetriever->dtoToPart($dto), $match); + + $this->commentHelper->setMessage(sprintf('Fetched data from %s on first page view', $match->getProviderName())); + $this->em->flush(); + } catch (\Throwable $exception) { + $this->logger->error('Could not fetch the data of part {id} from {provider} on page view: {message}', [ + 'id' => $part->getID(), + 'provider' => $match->getProviderKey(), + 'message' => $exception->getMessage(), + 'exception' => $exception, + ]); + + //Remember the failure, so the same failing request is not repeated on every page view + $failure = $this->partInfoCache->getItem($this->failureKey($part, $match)); + $failure->set(true); + $failure->expiresAfter(self::FAILURE_TTL); + $this->partInfoCache->save($failure); + + return self::STATUS_ERROR; + } finally { + $this->partInfoCache->deleteItem($lock_key); + } + + return self::STATUS_UPDATED; + } + + /** + * Returns the ID of the product at the provider. If the part was only recognized by its supplier, the provider + * is searched for the supplier part number, and the product must be found with exactly this number. + */ + private function resolveProviderId(OnViewMatch $match): string + { + if ($match->providerId !== null) { + return $match->providerId; + } + + $supplier_part_nr = (string) $match->supplierPartNr; + $results = $this->infoRetriever->searchByKeyword($supplier_part_nr, [$match->provider]); + + $result = $this->matcher->pickExactResult($match->provider, $supplier_part_nr, $results); + if ($result === null) { + throw new \RuntimeException(sprintf('%s has no (unambiguous) product with exactly the number "%s" (%d search results)', + $match->getProviderName(), $supplier_part_nr, count($results))); + } + + return $result->provider_id; + } + + /** + * Copies everything the target part is missing from the part built from the provider data. + */ + private function fillPart(Part $target, Part $provider_part, OnViewMatch $match): void + { + if ($target->getDescription() === '') { + $target->setDescription($provider_part->getDescription()); + } + if ($target->getComment() === '') { + $target->setComment($provider_part->getComment()); + } + if ($target->getManufacturerProductNumber() === '') { + $target->setManufacturerProductNumber($provider_part->getManufacturerProductNumber()); + } + if ($target->getCustomProductURL() === '') { + $target->setManufacturerProductURL($provider_part->getCustomProductURL()); + } + if ($target->getGtin() === null) { + $target->setGtin($provider_part->getGtin()); + } + if ($target->getMass() === null) { + $target->setMass($provider_part->getMass()); + } + if (in_array($target->getManufacturingStatus(), [null, ManufacturingStatus::NOT_SET], true) + && $provider_part->getManufacturingStatus() !== null) { + $target->setManufacturingStatus($provider_part->getManufacturingStatus()); + } + + if ($target->getManufacturer() === null && $provider_part->getManufacturer() !== null) { + //The manufacturer might be newly created for this part, so it has to be persisted explicitly + $this->em->persist($provider_part->getManufacturer()); + $target->setManufacturer($provider_part->getManufacturer()); + } + + $this->fillAttachments($target, $provider_part); + $this->fillParameters($target, $provider_part); + $this->fillOrderdetails($target, $provider_part, $match); + + //Mark the part as done, and allow to update it from the provider via the normal info provider tools later + $reference = $provider_part->getProviderReference(); + $target->setProviderReference(InfoProviderReference::providerReference( + (string) $reference->getProviderKey(), (string) $reference->getProviderId(), $reference->getProviderUrl() + )); + } + + private function fillAttachments(Part $target, Part $provider_part): void + { + $existing_urls = []; + foreach ($target->getAttachments() as $attachment) { + $existing_urls[] = $attachment->getURL(); + } + + $master = $provider_part->getMasterPictureAttachment(); + + /** @var Attachment $attachment */ + foreach ($provider_part->getAttachments()->toArray() as $attachment) { + if (in_array($attachment->getURL(), $existing_urls, true)) { + continue; + } + + //The attachment type might be newly created for this part, so it has to be persisted explicitly + if ($attachment->getAttachmentType() !== null) { + $this->em->persist($attachment->getAttachmentType()); + } + + $provider_part->removeAttachment($attachment); + $target->addAttachment($attachment); + + if ($attachment === $master && $target->getMasterPictureAttachment() === null) { + $target->setMasterPictureAttachment($attachment); + } + } + } + + private function fillParameters(Part $target, Part $provider_part): void + { + $existing_names = []; + foreach ($target->getParameters() as $parameter) { + $existing_names[] = mb_strtolower($parameter->getName()); + } + + /** @var AbstractParameter $parameter */ + foreach ($provider_part->getParameters()->toArray() as $parameter) { + if (in_array(mb_strtolower($parameter->getName()), $existing_names, true)) { + continue; + } + + $provider_part->removeParameter($parameter); + $target->addParameter($parameter); + } + } + + private function fillOrderdetails(Part $target, Part $provider_part, OnViewMatch $match): void + { + /** @var Orderdetail $orderdetail */ + foreach ($provider_part->getOrderdetails()->toArray() as $orderdetail) { + //Find the orderdetail of the target part, which describes the same product at the same supplier + $existing = null; + foreach ($target->getOrderdetails() as $target_orderdetail) { + if ($this->isSameOffer($target_orderdetail, $orderdetail, $match)) { + $existing = $target_orderdetail; + break; + } + } + + foreach ($orderdetail->getPricedetails() as $pricedetail) { + //The currency might be newly created for this part, so it has to be persisted explicitly + if ($pricedetail->getCurrency() !== null) { + $this->em->persist($pricedetail->getCurrency()); + } + } + + if ($existing === null) { + //The supplier might be newly created for this part, so it has to be persisted explicitly + if ($orderdetail->getSupplier() !== null) { + $this->em->persist($orderdetail->getSupplier()); + } + + $provider_part->removeOrderdetail($orderdetail); + $target->addOrderdetail($orderdetail); + continue; + } + + if ($existing->getSupplierPartNr() === '') { + $existing->setSupplierpartnr($orderdetail->getSupplierPartNr()); + } + if ($existing->getSupplierProductUrl() === '') { + $existing->setSupplierProductUrl($orderdetail->getSupplierProductUrl()); + } + + //Only add the current price if the part has no price for this product yet (e.g. the price it was bought for) + if ($existing->getPricedetails()->isEmpty()) { + foreach ($orderdetail->getPricedetails()->toArray() as $pricedetail) { + $orderdetail->removePricedetail($pricedetail); + $existing->addPricedetail($pricedetail); + } + $existing->setPricesIncludesVAT($orderdetail->getPricesIncludesVAT()); + } + } + } + + /** + * Checks if an orderdetail the part already has and one supplied by the provider are the same offer, so the + * part does not end up with two orderdetails for it. + */ + private function isSameOffer(Orderdetail $existing, Orderdetail $provided, OnViewMatch $match): bool + { + $existing_supplier = ProviderOnViewMatcher::normalizeName((string) $existing->getSupplier()?->getName()); + $provided_supplier = ProviderOnViewMatcher::normalizeName((string) $provided->getSupplier()?->getName()); + + $provided_id = $this->matcher->getProviderIdFromURL($match->provider, $provided->getSupplierProductUrl()); + + //The orderdetail the part was recognized by is the offer of this product at the store of the provider + if ($existing === $match->orderdetail && ($provided_id !== null || $existing_supplier === $provided_supplier)) { + return true; + } + + //Both link to the same product page + if ($provided_id !== null + && $this->matcher->getProviderIdFromURL($match->provider, $existing->getSupplierProductUrl()) === $provided_id) { + return true; + } + + //The same order number at the same supplier + return $existing_supplier !== '' && $existing_supplier === $provided_supplier + && ProviderOnViewMatcher::numbersEqual($existing->getSupplierPartNr(), $provided->getSupplierPartNr(), + [(string) $provided->getSupplier()?->getName()]); + } + + /** + * Counts a lookup against the daily limit of the provider and returns false if the limit is used up. + */ + private function consumeDailyLimit(string $provider_key): bool + { + $limit = $this->matcher->getDailyLimit($provider_key); + if ($limit <= 0) { + return true; + } + + $factory = new RateLimiterFactory([ + 'id' => $provider_key.'_on_view', + 'policy' => 'sliding_window', + 'limit' => $limit, + 'interval' => '24 hours', + ], new CacheStorage($this->partInfoCache)); + + return $factory->create('daily')->consume()->isAccepted(); + } + + private function failureKey(Part $part, OnViewMatch $match): string + { + //Provider keys are identifiers, but make sure that nothing the cache does not allow in a key gets in + return preg_replace('/[^A-Za-z0-9_.]/', '_', $match->getProviderKey()).'_on_view_failed_'.$part->getID(); + } +} diff --git a/src/Services/InfoProviderSystem/ProviderOnViewMatcher.php b/src/Services/InfoProviderSystem/ProviderOnViewMatcher.php new file mode 100644 index 000000000..279176536 --- /dev/null +++ b/src/Services/InfoProviderSystem/ProviderOnViewMatcher.php @@ -0,0 +1,334 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem; + +use App\Entity\Parts\Part; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Services\InfoProviderSystem\Providers\CanopyProvider; +use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; +use App\Services\InfoProviderSystem\Providers\URLHandlerInfoProviderInterface; +use App\Settings\InfoProviderSystem\CanopySettings; +use App\Settings\InfoProviderSystem\InfoProviderGeneralSettings; + +/** + * Decides which info provider can supply the data of a part, that has no info provider data yet, when its info page + * is opened ("fetch on view", see ProviderOnViewFetcher). + * + * A part is recognized by its orderdetails, in this order: + * 1. The product URL of an orderdetail is a product page of the provider. The URL then gives the ID of the product. + * 2. The supplier of an orderdetail is named like the provider and the orderdetail has a supplier part number. + * The product then has to be searched at the provider, and only a result with exactly this number is accepted. + * + * Nothing in here contacts a provider, so it can be used on every page view. + * + * @see \App\Tests\Services\InfoProviderSystem\ProviderOnViewMatcherTest + */ +final class ProviderOnViewMatcher +{ + public function __construct( + private readonly InfoProviderGeneralSettings $settings, + private readonly CanopySettings $canopySettings, + private readonly ProviderRegistry $providerRegistry, + ) { + } + + /** + * Returns the providers which fetch data when a part is viewed, in the order they were configured in. + * Providers which are not active (or do not exist) are ignored. + * @return array The providers indexed by their keys + */ + public function getEnabledProviders(): array + { + $keys = $this->settings->fetchOnViewProviders; + //The Canopy provider has its own switch, which exists longer than the general setting + if ($this->canopySettings->fetchOnView) { + $keys[] = CanopyProvider::PROVIDER_KEY; + } + + if ($keys === []) { + return []; + } + + $active = $this->providerRegistry->getActiveProviders(); + + $providers = []; + foreach ($keys as $key) { + if (is_string($key) && isset($active[$key])) { + $providers[$key] = $active[$key]; + } + } + + return $providers; + } + + /** + * Returns the maximum number of parts, which may be looked up at the given provider within 24 hours because + * their page was viewed. 0 means no limit. + */ + public function getDailyLimit(string $provider_key): int + { + if ($provider_key === CanopyProvider::PROVIDER_KEY) { + return $this->canopySettings->fetchOnViewDailyLimit; + } + + return $this->settings->fetchOnViewDailyLimit; + } + + /** + * Returns the ways the given part can be looked up at the enabled providers, best one first. This is empty for + * parts which already have info provider data, as only parts nobody looked up yet are filled when viewed. + * @return OnViewMatch[] + */ + public function findMatches(Part $part): array + { + //A provider reference means the part data already came from an info provider + if ($part->getProviderReference()->isProviderCreated()) { + return []; + } + + $providers = $this->getEnabledProviders(); + if ($providers === []) { + return []; + } + + $by_url = []; + $by_supplier = []; + + foreach ($part->getOrderdetails() as $orderdetail) { + //This includes URLs generated from the supplier's product URL template and the supplier part number + $url = $orderdetail->getSupplierProductUrl(); + $supplier_name = self::normalizeName((string) $orderdetail->getSupplier()?->getName()); + $supplier_part_nr = trim($orderdetail->getSupplierPartNr()); + + foreach ($providers as $provider) { + $id = $this->getProviderIdFromURL($provider, $url); + if ($id !== null) { + $by_url[] = new OnViewMatch($provider, $orderdetail, providerId: $id); + continue; + } + + if ($supplier_part_nr !== '' && $supplier_name !== '' + && $supplier_name === self::normalizeName($provider->getProviderInfo()->name)) { + $by_supplier[] = new OnViewMatch($provider, $orderdetail, supplierPartNr: $supplier_part_nr); + } + } + } + + return [...$by_url, ...$by_supplier]; + } + + /** + * Returns the ID the given provider uses for the product behind the given URL, or null if the URL is not a + * product page of the provider. + */ + public function getProviderIdFromURL(InfoProviderInterface $provider, ?string $url): ?string + { + if ($url === null || $url === '') { + return null; + } + + $host = parse_url($url, PHP_URL_HOST); + if (!is_string($host) || $host === '') { + return null; + } + + //Canopy does not handle URLs itself: its IDs are the ASINs, which are part of the Amazon product page URLs + if ($provider->getProviderInfo()->key === CanopyProvider::PROVIDER_KEY) { + //Canopy is queried for the configured marketplace only, an ASIN of another one would give wrong data + if (!self::isHostOfDomain($host, $this->canopySettings->getRealDomain())) { + return null; + } + + $path = (string) parse_url($url, PHP_URL_PATH); + if (preg_match('#/(?:dp|gp/product|gp/aw/d|exec/obidos/ASIN)/([A-Z0-9]{10})(?:[/?]|$)#', $path, $matches) === 1) { + return $matches[1]; + } + + return null; + } + + if (!$provider instanceof URLHandlerInfoProviderInterface) { + return null; + } + + foreach ($provider->getHandledDomains() as $domain) { + if (self::isHostOfDomain($host, (string) $domain)) { + $id = $provider->getIDFromURL($url); + + return $id === null || trim($id) === '' ? null : $id; + } + } + + return null; + } + + /** + * Picks the search result which is the product with the given supplier part number, or returns null if the + * results do not contain exactly this product. A result is never chosen because it is merely similar: the data + * is added to the part without anybody reviewing it, so no data is better than the data of the wrong product. + * + * A result is the product, if its provider ID, its manufacturer part number or one of its order numbers is the + * supplier part number (see numbersEqual()). If this is true for multiple results, they must all carry the same + * manufacturer part number (like the packaging variants of one product do), and then the first one is taken. + * + * @param SearchResultDTO[] $results The results of searching the provider for the supplier part number + */ + public function pickExactResult(InfoProviderInterface $provider, string $supplier_part_nr, array $results): ?SearchResultDTO + { + $info = $provider->getProviderInfo(); + $vendor_names = [$info->key, $info->name]; + + /** @var array $exact */ + $exact = []; + foreach ($results as $result) { + if (!$result instanceof SearchResultDTO || $result->provider_key !== $info->key) { + continue; + } + + $numbers = [$result->provider_id, $result->mpn]; + if ($result instanceof PartDetailDTO) { + foreach ($result->vendor_infos ?? [] as $vendor_info) { + $numbers[] = $vendor_info->order_number; + } + } + + foreach ($numbers as $number) { + if ($number !== null && self::numbersEqual($supplier_part_nr, $number, $vendor_names)) { + $exact[$result->provider_id] ??= $result; + break; + } + } + } + + if (count($exact) <= 1) { + return array_values($exact)[0] ?? null; + } + + //Multiple different products claim this number. That is only fine if they are the same article. + $mpns = []; + foreach ($exact as $result) { + $mpns[mb_strtolower(trim((string) $result->mpn))] = true; + } + if (count($mpns) !== 1 || isset($mpns[''])) { + return null; + } + + return array_values($exact)[0]; + } + + /** + * Checks if the two given part numbers are the same one. They are compared case-insensitively, and a vendor + * prefix in front of a numeric part number is ignored if the other number has none: "4062" is the same as + * "ADA4062" (a prefix taken from the name of the vendor) or "DEV-04062" (a prefix set apart by a separator). + * Two numbers with different prefixes (DEV-123 and PRT-123) are different. + * + * @param string[] $vendor_names The names of the vendor, a prefix has to be the beginning of one of them if + * it is not set apart from the number + */ + public static function numbersEqual(string $a, string $b, array $vendor_names = []): bool + { + $a = trim($a); + $b = trim($b); + if ($a === '' || $b === '') { + return false; + } + + if (mb_strtolower($a) === mb_strtolower($b)) { + return true; + } + + $bare_a = self::bareNumber($a); + $bare_b = self::bareNumber($b); + if ($bare_a !== null && $bare_b !== null) { + return $bare_a === $bare_b; + } + + //Otherwise exactly one of them must be a bare number, and the other one this number with a prefix + if ($bare_a !== null) { + return self::stripVendorPrefix($b, $vendor_names) === $bare_a; + } + if ($bare_b !== null) { + return self::stripVendorPrefix($a, $vendor_names) === $bare_b; + } + + return false; + } + + /** + * Returns the number without leading zeros (and without a leading #), if the given string is just a number. + */ + private static function bareNumber(string $value): ?string + { + if (preg_match('/^#?\s*0*(\d+)$/', $value, $matches) === 1) { + return $matches[1]; + } + + return null; + } + + /** + * Returns the number (without leading zeros) of a part number which consists of a vendor prefix and a number, + * or null if the given part number does not look like that. + * @param string[] $vendor_names + */ + private static function stripVendorPrefix(string $value, array $vendor_names): ?string + { + if (preg_match('/^([A-Za-z]{2,})([\s\-_#:.]*)0*(\d+)$/', $value, $matches) !== 1) { + return null; + } + + //A prefix followed by a separator is clearly no part of the number itself + if ($matches[2] !== '') { + return $matches[3]; + } + + //Without one, this could as well be a type designation (like KM100), so the prefix has to be the vendor + $prefix = strtolower($matches[1]); + foreach ($vendor_names as $vendor_name) { + if (str_starts_with(self::normalizeName($vendor_name), $prefix)) { + return $matches[3]; + } + } + + return null; + } + + /** + * Reduces a supplier or provider name to its letters and digits in lower case, so "Bambu Lab", "BambuLab" and + * "bambu-lab" are the same name. + */ + public static function normalizeName(string $name): string + { + return preg_replace('/[^\p{L}\p{N}]+/u', '', mb_strtolower($name)) ?? ''; + } + + private static function isHostOfDomain(string $host, string $domain): bool + { + $host = strtolower($host); + $domain = strtolower($domain); + + return $domain !== '' && ($host === $domain || str_ends_with($host, '.'.$domain)); + } +} diff --git a/src/Settings/InfoProviderSystem/CanopySettings.php b/src/Settings/InfoProviderSystem/CanopySettings.php index 9e6d8ed55..c84365827 100644 --- a/src/Settings/InfoProviderSystem/CanopySettings.php +++ b/src/Settings/InfoProviderSystem/CanopySettings.php @@ -30,7 +30,9 @@ use Jbtronics\SettingsBundle\Settings\SettingsParameter; use Jbtronics\SettingsBundle\Settings\SettingsTrait; use Symfony\Component\Form\Extension\Core\Type\ChoiceType; +use Symfony\Component\Form\Extension\Core\Type\NumberType; use Symfony\Component\Translation\TranslatableMessage as TM; +use Symfony\Component\Validator\Constraints as Assert; #[Settings(label: new TM("settings.ips.canopy"))] #[SettingsIcon("fa-plug")] @@ -79,6 +81,20 @@ class CanopySettings #[SettingsParameter(label: new TM("settings.ips.canopy.alwaysGetDetails"), description: new TM("settings.ips.canopy.alwaysGetDetails.help"))] public bool $alwaysGetDetails = false; + /** + * @var bool If true, an Amazon part without provider data is filled with data from Canopy when its info page is opened for the first time + */ + #[SettingsParameter(label: new TM("settings.ips.canopy.fetchOnView"), description: new TM("settings.ips.canopy.fetchOnView.help"))] + public bool $fetchOnView = false; + + /** + * @var int The maximum number of Canopy requests triggered by page views within 24 hours. 0 means no limit. + */ + #[SettingsParameter(label: new TM("settings.ips.canopy.fetchOnViewDailyLimit"), description: new TM("settings.ips.canopy.fetchOnViewDailyLimit.help"), + formType: NumberType::class, formOptions: ["scale" => 0, "attr" => ["min" => 0]])] + #[Assert\PositiveOrZero] + public int $fetchOnViewDailyLimit = 100; + /** * Returns the real domain (e.g. amazon.de) based on the selected domain (e.g. DE) * @return string diff --git a/src/Settings/InfoProviderSystem/InfoProviderGeneralSettings.php b/src/Settings/InfoProviderSystem/InfoProviderGeneralSettings.php index 1aa3e7d1b..e77e2e319 100644 --- a/src/Settings/InfoProviderSystem/InfoProviderGeneralSettings.php +++ b/src/Settings/InfoProviderSystem/InfoProviderGeneralSettings.php @@ -27,6 +27,7 @@ use App\Settings\SettingsIcon; use Jbtronics\SettingsBundle\ParameterTypes\ArrayType; use Jbtronics\SettingsBundle\ParameterTypes\StringType; +use Jbtronics\SettingsBundle\Metadata\EnvVarMode; use Jbtronics\SettingsBundle\Settings\Settings; use Jbtronics\SettingsBundle\Settings\SettingsParameter; use Symfony\Component\Translation\TranslatableMessage as TM; @@ -70,4 +71,35 @@ class InfoProviderGeneralSettings description: new TM("settings.ips.rate_limit_max_wait.help"))] #[Assert\Range(min: 0, max: 600)] public int $rateLimitMaxWait = 30; + + /** + * @var string[] The keys of the providers, which fill a part without info provider data when its info page is + * opened for the first time (see ProviderOnViewFetcher). Empty means that viewing a part never contacts a provider. + */ + #[SettingsParameter(type: ArrayType::class, label: new TM("settings.ips.fetch_on_view_providers"), + description: new TM("settings.ips.fetch_on_view_providers.help"), options: ['type' => StringType::class], + formType: ProviderSelectType::class, formOptions: ['input' => 'string', 'required' => false, 'empty_data' => []], + envVar: "PROVIDER_FETCH_ON_VIEW", envVarMode: EnvVarMode::OVERWRITE, envVarMapper: [self::class, 'mapFetchOnViewProvidersEnv'])] + public array $fetchOnViewProviders = []; + + /** + * @var int The maximum number of parts which are looked up at a single provider within 24 hours because their + * page was viewed. 0 means no limit. + */ + #[SettingsParameter(label: new TM("settings.ips.fetch_on_view_daily_limit"), + description: new TM("settings.ips.fetch_on_view_daily_limit.help"), + envVar: "int:PROVIDER_FETCH_ON_VIEW_DAILY_LIMIT", envVarMode: EnvVarMode::OVERWRITE)] + #[Assert\Range(min: 0, max: 100000)] + public int $fetchOnViewDailyLimit = 100; + + /** + * Turns the comma separated list of provider keys of the PROVIDER_FETCH_ON_VIEW environment variable into an array + * @return string[] + */ + public static function mapFetchOnViewProvidersEnv(string $providers): array + { + $keys = array_map(static fn(string $key): string => strtolower(trim($key)), explode(',', $providers)); + + return array_values(array_unique(array_filter($keys, static fn(string $key): bool => $key !== ''))); + } } diff --git a/templates/parts/info/_provider_fetch.html.twig b/templates/parts/info/_provider_fetch.html.twig new file mode 100644 index 000000000..848751188 --- /dev/null +++ b/templates/parts/info/_provider_fetch.html.twig @@ -0,0 +1,16 @@ +{# Shown on the first view of a part, while its data is retrieved from an info provider (see ProviderOnViewFetcher) #} +
+ + + + {{ 'part.info.provider_fetch.loading'|trans({'%provider%': provider_name}) }} +
diff --git a/templates/parts/info/show_part_info.html.twig b/templates/parts/info/show_part_info.html.twig index 5c613f3ba..8e7b12804 100644 --- a/templates/parts/info/show_part_info.html.twig +++ b/templates/parts/info/show_part_info.html.twig @@ -31,6 +31,10 @@ {% endblock %} {% block card_content %} + {% if fetch_on_view_provider is not null %} + {% include "parts/info/_provider_fetch.html.twig" with {provider_name: fetch_on_view_provider} %} + {% endif %} +
{% include "parts/info/_picture.html.twig" %} diff --git a/tests/Services/InfoProviderSystem/ProviderOnViewMatcherTest.php b/tests/Services/InfoProviderSystem/ProviderOnViewMatcherTest.php new file mode 100644 index 000000000..85354462a --- /dev/null +++ b/tests/Services/InfoProviderSystem/ProviderOnViewMatcherTest.php @@ -0,0 +1,419 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Tests\Services\InfoProviderSystem; + +use App\Entity\Parts\InfoProviderReference; +use App\Entity\Parts\Part; +use App\Entity\Parts\Supplier; +use App\Entity\PriceInformations\Orderdetail; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\ProviderInfoDTO; +use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Services\InfoProviderSystem\ProviderOnViewMatcher; +use App\Services\InfoProviderSystem\ProviderRegistry; +use App\Services\InfoProviderSystem\Providers\InfoProviderInterface; +use App\Services\InfoProviderSystem\Providers\URLHandlerInfoProviderInterface; +use App\Settings\InfoProviderSystem\CanopySettings; +use App\Settings\InfoProviderSystem\InfoProviderGeneralSettings; +use App\Tests\SettingsTestHelper; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\TestCase; + +final class ProviderOnViewMatcherTest extends TestCase +{ + private InfoProviderGeneralSettings $settings; + private CanopySettings $canopySettings; + + /** @var int The number of times a provider was asked for search results or details */ + private int $providerRequests = 0; + + protected function setUp(): void + { + $this->settings = SettingsTestHelper::createSettingsDummy(InfoProviderGeneralSettings::class); + $this->canopySettings = SettingsTestHelper::createSettingsDummy(CanopySettings::class); + $this->canopySettings->domain = 'US'; + $this->providerRequests = 0; + } + + /** + * A store provider, which recognizes the URLs https://www..com/product/ + */ + private function getStoreProvider(string $key, string $name, bool $active = true): InfoProviderInterface + { + $mock = $this->createMockForIntersectionOfInterfaces([InfoProviderInterface::class, URLHandlerInfoProviderInterface::class]); + $mock->method('getProviderInfo')->willReturn(new ProviderInfoDTO(key: $key, name: $name)); + $mock->method('isActive')->willReturn($active); + $mock->method('getHandledDomains')->willReturn(["$key.com"]); + $mock->method('getIDFromURL')->willReturnCallback( + static fn(string $url): ?string => preg_match('#/product/(\w+)#', $url, $matches) === 1 ? $matches[1] : null + ); + $this->countRequests($mock); + + return $mock; + } + + /** + * A provider which does not handle URLs (like the API of a distributor) + */ + private function getPlainProvider(string $key, string $name): InfoProviderInterface + { + $mock = $this->createMock(InfoProviderInterface::class); + $mock->method('getProviderInfo')->willReturn(new ProviderInfoDTO(key: $key, name: $name)); + $mock->method('isActive')->willReturn(true); + $this->countRequests($mock); + + return $mock; + } + + private function countRequests(InfoProviderInterface $mock): void + { + $mock->method('searchByKeyword')->willReturnCallback(function (): array { + $this->providerRequests++; + return []; + }); + $mock->method('getDetails')->willReturnCallback(function (): never { + $this->providerRequests++; + throw new \RuntimeException('The matcher must not request details'); + }); + } + + /** + * @param string[] $enabled The keys of the providers "fetch on view" is enabled for + */ + private function getMatcher(array $enabled, ?array $providers = null): ProviderOnViewMatcher + { + $this->settings->fetchOnViewProviders = $enabled; + + return new ProviderOnViewMatcher($this->settings, $this->canopySettings, new ProviderRegistry($providers ?? [ + $this->getStoreProvider('bambulab', 'Bambu Lab'), + $this->getStoreProvider('sparkfun', 'SparkFun'), + $this->getStoreProvider('adafruit', 'Adafruit'), + $this->getStoreProvider('pololu', 'Pololu', active: false), + $this->getPlainProvider('canopy', 'Amazon (Canopy)'), + $this->getPlainProvider('distributor', 'Distributor'), + ])); + } + + private function getPart(?string $supplier, string $supplier_part_nr = '', string $url = ''): Part + { + $part = new Part(); + $part->setName('Test part'); + $this->addOrderdetail($part, $supplier, $supplier_part_nr, $url); + + return $part; + } + + private function addOrderdetail(Part $part, ?string $supplier, string $supplier_part_nr = '', string $url = ''): Orderdetail + { + $orderdetail = new Orderdetail(); + if ($supplier !== null) { + $orderdetail->setSupplier((new Supplier())->setName($supplier)); + } + $orderdetail->setSupplierpartnr($supplier_part_nr); + $orderdetail->setSupplierProductUrl($url); + $part->addOrderdetail($orderdetail); + + return $orderdetail; + } + + private function searchResult(string $provider_id, ?string $mpn = null, string $provider_key = 'sparkfun'): SearchResultDTO + { + return new SearchResultDTO(provider_key: $provider_key, provider_id: $provider_id, name: 'Product '.$provider_id, + description: '', mpn: $mpn); + } + + public function testNothingEnabledByDefault(): void + { + $matcher = $this->getMatcher([]); + + $this->assertSame([], $matcher->getEnabledProviders()); + $this->assertSame([], $matcher->findMatches($this->getPart('Adafruit', '4062', 'https://www.adafruit.com/product/4062'))); + } + + public function testEnabledProvidersKeepOrderAndSkipInactiveAndUnknown(): void + { + $matcher = $this->getMatcher(['sparkfun', 'pololu', 'does_not_exist', 'bambulab']); + + $this->assertSame(['sparkfun', 'bambulab'], array_keys($matcher->getEnabledProviders())); + } + + public function testCanopyIsEnabledByItsOwnSetting(): void + { + $matcher = $this->getMatcher(['adafruit']); + $this->assertSame(['adafruit'], array_keys($matcher->getEnabledProviders())); + + $this->canopySettings->fetchOnView = true; + $this->assertSame(['adafruit', 'canopy'], array_keys($matcher->getEnabledProviders())); + } + + public function testDailyLimit(): void + { + $matcher = $this->getMatcher(['adafruit']); + $this->settings->fetchOnViewDailyLimit = 7; + $this->canopySettings->fetchOnViewDailyLimit = 3; + + $this->assertSame(7, $matcher->getDailyLimit('adafruit')); + //Canopy is billed per request, so it keeps the limit of its own settings + $this->assertSame(3, $matcher->getDailyLimit('canopy')); + } + + public function testMatchByProductURL(): void + { + $matcher = $this->getMatcher(['adafruit', 'sparkfun']); + //The supplier does not matter, if the URL is a product page of the provider + $part = $this->getPart('Some reseller', 'XYZ', 'https://www.adafruit.com/product/4062'); + + $matches = $matcher->findMatches($part); + + $this->assertCount(1, $matches); + $this->assertSame('adafruit', $matches[0]->getProviderKey()); + $this->assertSame('Adafruit', $matches[0]->getProviderName()); + $this->assertSame('4062', $matches[0]->providerId); + $this->assertNull($matches[0]->supplierPartNr); + $this->assertSame($part->getOrderdetails()->first(), $matches[0]->orderdetail); + } + + public function testURLOfOtherDomainOrOtherPageDoesNotMatch(): void + { + $matcher = $this->getMatcher(['adafruit']); + + //Looks like a product URL, but is another website + $this->assertSame([], $matcher->findMatches($this->getPart(null, '', 'https://www.notadafruit.com/product/4062'))); + $this->assertSame([], $matcher->findMatches($this->getPart(null, '', 'https://adafruit.com.example.org/product/4062'))); + //The right website, but no product page + $this->assertSame([], $matcher->findMatches($this->getPart(null, '', 'https://www.adafruit.com/category/17'))); + //Not an URL at all + $this->assertSame([], $matcher->findMatches($this->getPart(null, '', 'adafruit.com/product/4062'))); + } + + public function testMatchBySupplierNameAndPartNumber(): void + { + $matcher = $this->getMatcher(['bambulab', 'sparkfun']); + + $matches = $matcher->findMatches($this->getPart('Bambu Lab', ' 32101 ')); + $this->assertCount(1, $matches); + $this->assertSame('bambulab', $matches[0]->getProviderKey()); + $this->assertNull($matches[0]->providerId); + $this->assertSame('32101', $matches[0]->supplierPartNr); + + //The name is compared without case, spaces and punctuation + $this->assertSame('bambulab', $matcher->findMatches($this->getPart('BAMBULAB', '32101'))[0]->getProviderKey()); + $this->assertSame('sparkfun', $matcher->findMatches($this->getPart('Spark-Fun', 'DEV-13975'))[0]->getProviderKey()); + } + + public function testSupplierNameAloneIsNotEnough(): void + { + $matcher = $this->getMatcher(['bambulab', 'sparkfun']); + + //No supplier part number to look up + $this->assertSame([], $matcher->findMatches($this->getPart('Bambu Lab', ''))); + //Not the name of the provider + $this->assertSame([], $matcher->findMatches($this->getPart('Bambu Lab Europe', '32101'))); + //No supplier + $this->assertSame([], $matcher->findMatches($this->getPart(null, '32101'))); + //The provider of this supplier is not enabled + $this->assertSame([], $matcher->findMatches($this->getPart('Adafruit', '4062'))); + //No orderdetail at all + $this->assertSame([], $matcher->findMatches((new Part())->setName('Test'))); + } + + public function testURLMatchesComeBeforeSupplierMatches(): void + { + $matcher = $this->getMatcher(['bambulab', 'adafruit']); + + $part = $this->getPart('Bambu Lab', '32101'); + $this->addOrderdetail($part, 'Somebody', '', 'https://www.adafruit.com/product/4062'); + + $matches = $matcher->findMatches($part); + $this->assertCount(2, $matches); + $this->assertSame('adafruit', $matches[0]->getProviderKey()); + $this->assertSame('4062', $matches[0]->providerId); + $this->assertSame('bambulab', $matches[1]->getProviderKey()); + } + + public function testOrderdetailWithURLAndSupplierGivesOnlyTheURLMatch(): void + { + $matcher = $this->getMatcher(['adafruit']); + + $matches = $matcher->findMatches($this->getPart('Adafruit', 'ADA4062', 'https://www.adafruit.com/product/4062')); + $this->assertCount(1, $matches); + $this->assertSame('4062', $matches[0]->providerId); + } + + public function testPartWithProviderReferenceIsNotTouched(): void + { + $matcher = $this->getMatcher(['adafruit']); + + $part = $this->getPart('Adafruit', '4062', 'https://www.adafruit.com/product/4062'); + $part->setProviderReference(InfoProviderReference::providerReference('distributor', 'ABC')); + + $this->assertSame([], $matcher->findMatches($part)); + } + + public function testCanopyMatchesAmazonURLsOfTheConfiguredMarketplace(): void + { + $this->canopySettings->fetchOnView = true; + $matcher = $this->getMatcher([]); + + $matches = $matcher->findMatches($this->getPart('Amazon', '', 'https://www.amazon.com/Some-Product/dp/B00EXAMPLE?th=1')); + $this->assertCount(1, $matches); + $this->assertSame('canopy', $matches[0]->getProviderKey()); + $this->assertSame('B00EXAMPLE', $matches[0]->providerId); + + $this->assertSame('B00EXAMPLE', $matcher->findMatches($this->getPart(null, '', 'https://smile.amazon.com/gp/product/B00EXAMPLE'))[0]->providerId); + + //Another marketplace, no product page, and a supplier named Amazon without an URL + $this->assertSame([], $matcher->findMatches($this->getPart('Amazon', '', 'https://www.amazon.de/dp/B00EXAMPLE'))); + $this->assertSame([], $matcher->findMatches($this->getPart('Amazon', '', 'https://www.amazon.com/gp/help/customer'))); + $this->assertSame([], $matcher->findMatches($this->getPart('Amazon', 'B00EXAMPLE'))); + } + + public function testCanopyIsNotUsedIfDisabled(): void + { + $matcher = $this->getMatcher(['adafruit']); + + $this->assertSame([], $matcher->findMatches($this->getPart('Amazon', '', 'https://www.amazon.com/dp/B00EXAMPLE'))); + } + + public function testProviderWithoutURLHandlingMatchesBySupplierOnly(): void + { + $matcher = $this->getMatcher(['distributor']); + + $this->assertSame([], $matcher->findMatches($this->getPart(null, '', 'https://www.distributor.com/product/1'))); + $this->assertCount(1, $matcher->findMatches($this->getPart('Distributor', 'ABC-1'))); + } + + public function testMatchingDoesNotContactTheProviders(): void + { + $matcher = $this->getMatcher(['bambulab', 'sparkfun', 'adafruit', 'distributor']); + + $part = $this->getPart('Bambu Lab', '32101'); + $this->addOrderdetail($part, 'Adafruit', '4062', 'https://www.adafruit.com/product/4062'); + $this->addOrderdetail($part, 'Distributor', 'ABC'); + + $this->assertCount(3, $matcher->findMatches($part)); + $this->assertSame(0, $this->providerRequests); + } + + public static function numbersProvider(): \Generator + { + yield 'identical' => [true, '32101', '32101']; + yield 'case' => [true, 'dev-13975', 'DEV-13975']; + yield 'whitespace' => [true, ' KM100 ', 'KM100']; + yield 'leading zeros' => [true, '08809', '8809']; + yield 'hash' => [true, '#4062', '4062']; + yield 'vendor prefix from the name' => [true, 'ADA4062', '4062', ['adafruit', 'Adafruit']]; + yield 'vendor prefix on the other side' => [true, '4062', 'ada4062', ['adafruit', 'Adafruit']]; + yield 'prefix with separator' => [true, '13975', 'DEV-13975']; + yield 'prefix with separator and zeros' => [true, '8809', 'PRT-08809']; + + yield 'empty' => [false, '', '']; + yield 'one empty' => [false, '4062', '']; + yield 'different numbers' => [false, '4062', '4063']; + yield 'number contained' => [false, '4062', '14062']; + yield 'different prefixes' => [false, 'DEV-13975', 'PRT-13975']; + yield 'prefix is not the vendor' => [false, 'KM100', '100', ['thorlabs', 'Thorlabs']]; + yield 'prefix without vendor names' => [false, 'ADA4062', '4062']; + yield 'suffix' => [false, '4062-A', '4062']; + yield 'similar' => [false, 'KM100', 'KM100T']; + } + + #[DataProvider('numbersProvider')] + public function testNumbersEqual(bool $expected, string $a, string $b, array $vendor_names = []): void + { + $this->assertSame($expected, ProviderOnViewMatcher::numbersEqual($a, $b, $vendor_names)); + $this->assertSame($expected, ProviderOnViewMatcher::numbersEqual($b, $a, $vendor_names)); + } + + public function testPickExactResultByProviderIdMpnOrOrderNumber(): void + { + $matcher = $this->getMatcher(['sparkfun']); + $provider = $matcher->getEnabledProviders()['sparkfun']; + + //By provider ID, with the vendor prefix ignored. The fuzzy neighbours are not taken. + $results = [$this->searchResult('DEV-13976'), $this->searchResult('DEV-13975'), $this->searchResult('DEV-139750')]; + $this->assertSame($results[1], $matcher->pickExactResult($provider, '13975', $results)); + $this->assertSame($results[1], $matcher->pickExactResult($provider, 'dev-13975', $results)); + + //By manufacturer part number + $results = [$this->searchResult('some-slug/1', '32100'), $this->searchResult('other-slug/2', '32101')]; + $this->assertSame($results[1], $matcher->pickExactResult($provider, '32101', $results)); + + //By the order number of the store + $detail = new PartDetailDTO(provider_key: 'sparkfun', provider_id: 'slug', name: 'Product', description: '', + vendor_infos: [new PurchaseInfoDTO('SparkFun', 'SKU-777', [])]); + $this->assertSame($detail, $matcher->pickExactResult($provider, 'SKU-777', [$this->searchResult('other'), $detail])); + } + + public function testPickExactResultNeverGuesses(): void + { + $matcher = $this->getMatcher(['sparkfun']); + $provider = $matcher->getEnabledProviders()['sparkfun']; + + //No results at all + $this->assertNull($matcher->pickExactResult($provider, '13975', [])); + //A single result is not taken just because it is the only one + $this->assertNull($matcher->pickExactResult($provider, '13975', [$this->searchResult('DEV-13976', 'DEV-13976')])); + //Similar numbers + $this->assertNull($matcher->pickExactResult($provider, '1397', [$this->searchResult('DEV-13975'), $this->searchResult('DEV-01398')])); + //The number only appears in the name + $this->assertNull($matcher->pickExactResult($provider, '13975', [ + new SearchResultDTO(provider_key: 'sparkfun', provider_id: 'DEV-1', name: 'Replacement for 13975', description: '13975'), + ])); + //A result of another provider + $this->assertNull($matcher->pickExactResult($provider, '13975', [$this->searchResult('13975', provider_key: 'adafruit')])); + } + + public function testPickExactResultWithMultipleExactResults(): void + { + $matcher = $this->getMatcher(['bambulab']); + $provider = $matcher->getEnabledProviders()['bambulab']; + + //The packaging variants of one article share its number: the first one (the best match of the provider) is taken + $results = [ + $this->searchResult('petg-translucent/1', '32101', 'bambulab'), + $this->searchResult('petg-translucent/2', '32101', 'bambulab'), + $this->searchResult('petg-translucent/3', '32102', 'bambulab'), + ]; + $this->assertSame($results[0], $matcher->pickExactResult($provider, '32101', $results)); + + //The same result listed twice is still one product + $results = [$this->searchResult('4062', null, 'bambulab'), $this->searchResult('4062', null, 'bambulab')]; + $this->assertSame($results[0], $matcher->pickExactResult($provider, '4062', $results)); + + //Different products claiming the number (one by its ID, one by its part number) are ambiguous + $results = [$this->searchResult('32101', 'ABC', 'bambulab'), $this->searchResult('other', '32101', 'bambulab')]; + $this->assertNull($matcher->pickExactResult($provider, '32101', $results)); + $results = [$this->searchResult('32101', null, 'bambulab'), $this->searchResult('BBL-32101', null, 'bambulab')]; + $this->assertNull($matcher->pickExactResult($provider, '32101', $results)); + } + + public function testEnvVarMapper(): void + { + $this->assertSame(['bambulab', 'sparkfun', 'adafruit'], + InfoProviderGeneralSettings::mapFetchOnViewProvidersEnv(' bambulab, SparkFun ,,adafruit,bambulab')); + $this->assertSame([], InfoProviderGeneralSettings::mapFetchOnViewProvidersEnv('')); + } +} diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 6dd3538d0..68b45f0b7 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -15806,5 +15806,83 @@ Buerklin-API Authentication server: The minimum time in seconds between two requests to the Bambu Lab store (API and product pages). Requests which come too early are delayed, so a lookup can take a while. If the store refuses a request (HTTP 403, 429 or 503), the provider sends no requests at all for one hour. 0 disables the delay. + + + settings.ips.canopy.fetchOnView + Fetch data when a part is viewed + + + + + settings.ips.canopy.fetchOnView.help + When selected, an Amazon part without info provider data is filled with data from Canopy the first time its info page is opened. This way only the parts somebody actually looks at cause (billed) API requests. Only missing data is added, existing data of the part is never changed. Everybody who can view a part triggers this. + + + + + settings.ips.canopy.fetchOnViewDailyLimit + Max. requests per day when viewing parts + + + + + settings.ips.canopy.fetchOnViewDailyLimit.help + The maximum number of Canopy requests which viewing parts may cause within 24 hours. Use 0 for no limit. + + + + + part.info.provider_fetch.loading + Fetching product data from %provider%… + + + + + part.info.provider_fetch.updated + Product data received. Loading the updated part… + + + + + part.info.provider_fetch.limit + Product data was not fetched from %provider%, because the daily request limit is reached. + + + + + part.info.provider_fetch.error + Product data could not be fetched from %provider%. + + + + + part.info.provider_fetch.flash.updated + This part was filled with product data from %provider%. + + + + + settings.ips.fetch_on_view_providers + Fetch data when a part is viewed + + + + + settings.ips.fetch_on_view_providers.help + A part without info provider data is filled with data from the providers selected here, the first time its info page is opened. A part is recognized by an orderdetail which either links to a product page of the provider, or whose supplier is named like the provider and has a supplier part number the provider knows exactly. Only missing data is added, existing data of the part is never changed. Everybody who can view a part triggers this. Leave empty to never contact a provider when viewing a part. + + + + + settings.ips.fetch_on_view_daily_limit + Max. lookups per day when viewing parts + + + + + settings.ips.fetch_on_view_daily_limit.help + The maximum number of parts which viewing them may look up at each of the providers above within 24 hours. Use 0 for no limit. The Canopy provider has its own limit in its settings. + +