diff --git a/lib/private/Files/SetupManager.php b/lib/private/Files/SetupManager.php
index 393a6a7de0fc6..ca7f69c2175b0 100644
--- a/lib/private/Files/SetupManager.php
+++ b/lib/private/Files/SetupManager.php
@@ -69,6 +69,8 @@ class SetupManager {
 	private array $setupUsers = [];
 	// List of users for which all mounts are setup
 	private array $setupUsersComplete = [];
+	// List of users for which we've already refreshed the non-authoritative mounts
+	private array $usersMountsUpdated = [];
 	/**
 	 * An array of provider classes that have been set up, indexed by UserUID.
 	 *
@@ -233,6 +235,10 @@ function ($mountPoint, IStorage $storage, IMountPoint $mount) use ($reSharingEna
 	 * Update the cached mounts for all non-authoritative mount providers for a user.
 	 */
 	private function updateNonAuthoritativeProviders(IUser $user): void {
+		if (isset($this->usersMountsUpdated[$user->getUID()])) {
+			return;
+		}
+
 		// prevent recursion loop from when getting mounts from providers ends up setting up the filesystem
 		static $updatingProviders = false;
 		if ($updatingProviders) {
@@ -253,6 +259,7 @@ private function updateNonAuthoritativeProviders(IUser $user): void {
 		$mount = $this->mountProviderCollection->getUserMountsForProviderClasses($user, $providerNames);
 		$this->userMountCache->registerMounts($user, $mount, $providerNames);
 
+		$this->usersMountsUpdated[$user->getUID()] = true;
 		$updatingProviders = false;
 	}
 
@@ -672,8 +679,13 @@ public function setupForProvider(string $path, array $providers): void {
 		}
 
 		if ($this->fullSetupRequired($user)) {
-			$this->setupForUser($user);
-			return;
+			if ($this->optimizeAuthoritativeProviders) {
+				$this->updateNonAuthoritativeProviders($user);
+				$this->markUserMountsCached($user);
+			} else {
+				$this->setupForUser($user);
+				return;
+			}
 		}
 
 		$this->eventLogger->start('fs:setup:user:providers', 'Setup filesystem for ' . implode(', ', $providers));
@@ -717,6 +729,7 @@ public function tearDown() {
 		$this->setupUserMountProviders = [];
 		$this->setupMountProviderPaths = [];
 		$this->fullSetupRequired = [];
+		$this->usersMountsUpdated = [];
 		$this->rootSetup = false;
 		$this->mountManager->clear();
 		$this->userMountCache->clear();