feat: isolate account-bound proxy leases

README.md

@@ -352,7 +352,7 @@ Egress nodes are scoped to Build, Web, Console, or Web assets. The admin console
 - Proxy-pool mode without global cooldown after one connection failure
 - Immediate recovery probes after fixed-proxy transport failures, with per-node coalescing and bounded waiting for fast retry
 - Optional [Egress Quality Guard](./tools/egress-quality-guard/README.md) for active per-node model probes, guarded quarantine, and recovery; enable it with the built-in `quality-guard` Compose profile
-- Give each sticky session its own fixed node (`proxyPool=false`). Do not merge several stickies into one node, or the guard can only quarantine the whole group
+- Nodes whose proxy username contains `{account}` are treated as lease-scoped: a passive anomaly temporarily removes only the audited account lease, then recovery pins the probe to that same account and node. An unhealthy probe renews the hold; an expired hold no longer blocks routing if the sidecar is unavailable, so stale guard state cannot strand an account indefinitely. The shared node is never disabled and the rendered proxy identity is never exposed. Ordinary fixed sticky sessions can still be managed as separate nodes
 
 Hysteria and TUIC are not supported yet. FlareSolverr accepts only HTTP/SOCKS proxy URLs, so automatic clearance refresh cannot use a tunnel share URL directly.
 

README.zh-CN.md

@@ -350,7 +350,7 @@ curl http://127.0.0.1:8000/v1/responses \
 - 代理池模式，单次连接失败不会触发全局冷却
 - 固定代理传输失败后立即复测；同节点复测自动合并，后续绑定请求限时等待并在恢复后快速重试
 - 可选的[出口质量守护程序](./tools/egress-quality-guard/README.zh-CN.md)，支持逐节点模型探测、防误杀隔离和自动恢复；通过内置的 `quality-guard` Compose profile 按需启用
-- 固定 sticky 会话应各自建成独立节点（`proxyPool=false`）。不要把多条 sticky 合成一个节点，否则质量守护只能整组摘流，无法定位坏会话
+- 代理用户名包含 `{account}` 的节点会被识别为租约级节点：被动审计异常只会临时移出对应的账号租约，冷却后固定使用同一账号和节点复测，复测异常会续期；若 sidecar 不可用，已到期的隔离不会继续阻断路由，避免孤儿状态永久卡住账号。共享节点始终不会因此停用，也不会暴露渲染后的代理身份。普通固定 sticky 会话仍可按独立节点管理
 
 Hysteria 与 TUIC 暂未支持。FlareSolverr 仅接受 HTTP/SOCKS 代理地址，因此自动刷新 Clearance 暂不能直接使用隧道分享链接。
 

backend/internal/application/egress/service.go

@@ -25,6 +25,8 @@ var (
 	ErrProbeStale              = errors.New("代理配置在探测期间已更新，请重新测试")
 	ErrQualityProbeUnavailable = errors.New("出口质量探测不可用")
 	ErrQualityProbeNoAccount   = errors.New("质量检测暂无可调度账号")
+	ErrQualityLeaseUnavailable = errors.New("租约级质量隔离不可用")
+	ErrQualityLeaseConflict    = errors.New("租约状态已变化")
 	ErrClearanceUnavailable    = errors.New("Clearance 刷新不可用")
 	ErrProxyProfileUnavailable = errors.New("共享代理配置功能不可用")
 	ErrProxyProfileInUse       = errors.New("共享代理配置仍被节点使用")
@@ -42,6 +44,7 @@ const (
 
 type QualityProbeInput struct {
 	ClientKeyID     uint64
+	AccountID       uint64
 	Model           string
 	Prompt          string
 	Expected        string
@@ -116,6 +119,7 @@ type Service struct {
 	repository                  ServiceRepository
 	proxyProfiles               repository.EgressProxyProfileRepository
 	accounts                    AccountBindingRepository
+	qualityLeases               QualityLeaseRepository
 	operations                  OperationsRepository
 	cipher                      *security.Cipher
 	mu                          sync.RWMutex
@@ -131,6 +135,25 @@ type Service struct {
 	autoAssignMaxMigrationShare float64
 }
 
+// QualityLeaseRepository is optional and deliberately separate from account
+// binding administration. It exposes only the state required to isolate one
+// account-bound proxy lease.
+type QualityLeaseRepository interface {
+	Get(context.Context, uint64) (accountdomain.Credential, error)
+	ListEgressLeaseBlocks(context.Context, int, *accountdomain.EgressLeaseBlockCursor) ([]accountdomain.EgressLeaseBlock, error)
+	UpsertEgressLeaseBlock(context.Context, accountdomain.EgressLeaseBlock) (accountdomain.EgressLeaseBlock, error)
+	DeleteEgressLeaseBlock(context.Context, uint64, uint64, string) (bool, error)
+	DeleteEgressLeaseBlocksByNodes(context.Context, []uint64) (int64, error)
+	PruneInvalidEgressLeaseBlocks(context.Context, int) (int64, error)
+}
+
+type QualityLeaseInput struct {
+	AccountID         uint64
+	NodeID            uint64
+	Reason            string
+	QuarantineSeconds int
+}
+
 func (s *Service) SetQualityProber(value QualityProber) {
 	s.mu.Lock()
 	s.qualityProber = value
@@ -174,6 +197,15 @@ func (s *Service) ProbeQuality(ctx context.Context, nodeID uint64, input Quality
 	if node.Scope != domain.ScopeBuild || strings.TrimSpace(node.EncryptedProxyURL) == "" {
 		return QualityProbeResult{}, fmt.Errorf("%w: 质量探测仅支持已配置代理的 grok_build 节点", ErrInvalidInput)
 	}
+	if input.AccountID != 0 {
+		if !s.accountBoundProxy(node) || s.qualityLeases == nil {
+			return QualityProbeResult{}, fmt.Errorf("%w: 账号定向探测仅支持按账号派生代理的节点", ErrInvalidInput)
+		}
+		credential, loadErr := s.qualityLeases.Get(ctx, input.AccountID)
+		if loadErr != nil || credential.Provider != accountdomain.ProviderBuild || !credential.Enabled || credential.AuthStatus != accountdomain.AuthStatusActive || credential.EgressNodeID != nodeID {
+			return QualityProbeResult{}, ErrQualityProbeNoAccount
+		}
+	}
 	s.mu.RLock()
 	prober := s.qualityProber
 	s.mu.RUnlock()
@@ -241,10 +273,111 @@ func NewService(storage ServiceRepository, cipher *security.Cipher, browserUA st
 	}
 	if len(accounts) > 0 {
 		service.accounts = accounts[0]
+		if leases, ok := accounts[0].(QualityLeaseRepository); ok {
+			service.qualityLeases = leases
+		}
 	}
 	return service
 }
 
+func (s *Service) ListQualityLeases(ctx context.Context, limit int, cursor *accountdomain.EgressLeaseBlockCursor) ([]accountdomain.EgressLeaseBlock, error) {
+	if s.qualityLeases == nil {
+		return nil, ErrQualityLeaseUnavailable
+	}
+	if limit < 1 || limit > 1001 {
+		return nil, ErrInvalidInput
+	}
+	if cursor == nil {
+		for range 10 {
+			pruned, err := s.qualityLeases.PruneInvalidEgressLeaseBlocks(ctx, 1000)
+			if err != nil {
+				return nil, err
+			}
+			if pruned < 1000 {
+				break
+			}
+		}
+		nodes, err := s.repository.ListEgressNodes(ctx, domain.ScopeBuild, repository.SortQuery{})
+		if err != nil {
+			return nil, err
+		}
+		staleNodeIDs := make([]uint64, 0)
+		for _, node := range nodes {
+			if !node.Enabled || !s.accountBoundProxy(node) {
+				staleNodeIDs = append(staleNodeIDs, node.ID)
+			}
+		}
+		if _, err := s.qualityLeases.DeleteEgressLeaseBlocksByNodes(ctx, staleNodeIDs); err != nil {
+			return nil, err
+		}
+	}
+	return s.qualityLeases.ListEgressLeaseBlocks(ctx, limit, cursor)
+}
+
+func (s *Service) QuarantineQualityLease(ctx context.Context, input QualityLeaseInput) (accountdomain.EgressLeaseBlock, error) {
+	input.Reason = strings.TrimSpace(input.Reason)
+	if input.AccountID == 0 || input.NodeID == 0 || input.QuarantineSeconds < 30 || input.QuarantineSeconds > 86400 || !qualityLeaseReasonAllowed(input.Reason) {
+		return accountdomain.EgressLeaseBlock{}, ErrInvalidInput
+	}
+	if s.qualityLeases == nil {
+		return accountdomain.EgressLeaseBlock{}, ErrQualityLeaseUnavailable
+	}
+	node, err := s.repository.GetEgressNode(ctx, input.NodeID)
+	if errors.Is(err, repository.ErrNotFound) {
+		return accountdomain.EgressLeaseBlock{}, ErrNotFound
+	}
+	if err != nil {
+		return accountdomain.EgressLeaseBlock{}, err
+	}
+	if !node.Enabled || node.Scope != domain.ScopeBuild || !s.accountBoundProxy(node) {
+		return accountdomain.EgressLeaseBlock{}, ErrInvalidInput
+	}
+	credential, err := s.qualityLeases.Get(ctx, input.AccountID)
+	if err != nil || credential.Provider != accountdomain.ProviderBuild || !credential.Enabled || credential.AuthStatus != accountdomain.AuthStatusActive || credential.EgressNodeID != input.NodeID {
+		return accountdomain.EgressLeaseBlock{}, ErrQualityLeaseConflict
+	}
+	version, err := security.NewOpaqueToken(18)
+	if err != nil {
+		return accountdomain.EgressLeaseBlock{}, err
+	}
+	now := time.Now().UTC()
+	value, err := s.qualityLeases.UpsertEgressLeaseBlock(ctx, accountdomain.EgressLeaseBlock{
+		AccountID: input.AccountID, NodeID: input.NodeID, Reason: input.Reason, Version: version,
+		CooldownUntil: now.Add(time.Duration(input.QuarantineSeconds) * time.Second), UpdatedAt: now,
+	})
+	if errors.Is(err, repository.ErrConflict) || errors.Is(err, repository.ErrNotFound) {
+		return accountdomain.EgressLeaseBlock{}, ErrQualityLeaseConflict
+	}
+	return value, err
+}
+
+func (s *Service) RestoreQualityLease(ctx context.Context, accountID, nodeID uint64, version string) (bool, error) {
+	version = strings.TrimSpace(version)
+	if accountID == 0 || nodeID == 0 || version == "" || len(version) > 64 {
+		return false, ErrInvalidInput
+	}
+	if s.qualityLeases == nil {
+		return false, ErrQualityLeaseUnavailable
+	}
+	restored, err := s.qualityLeases.DeleteEgressLeaseBlock(ctx, accountID, nodeID, version)
+	if err != nil {
+		return false, err
+	}
+	if !restored {
+		return false, ErrQualityLeaseConflict
+	}
+	return true, nil
+}
+
+func qualityLeaseReasonAllowed(value string) bool {
+	switch value {
+	case "hard_tps", "soft_tps", "buffered_burst", "missing_thinking", "expected_marker_missing", "insufficient_output_tokens", "insufficient_generation_window", "probe_errors", "recovery_probe_error", "rotation_error":
+		return true
+	default:
+		return false
+	}
+}
+
 func (s *Service) UpdateDefaults(browserUA string) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -368,6 +501,7 @@ func (s *Service) Update(ctx context.Context, id uint64, input Input) (domain.Pu
 		return domain.PublicNode{}, err
 	}
 	previousScope := value.Scope
+	previousProxyURL := value.EncryptedProxyURL
 	value, err = s.applyInput(value, input, false)
 	if err != nil {
 		return domain.PublicNode{}, err
@@ -386,6 +520,11 @@ func (s *Service) Update(ctx context.Context, id uint64, input Input) (domain.Pu
 	}
 	if err == nil {
 		s.forgetClearance(updated.ID)
+		if !updated.Enabled || previousScope != updated.Scope || previousProxyURL != updated.EncryptedProxyURL {
+			if clearErr := s.clearQualityLeasesForNodes(ctx, []uint64{updated.ID}); clearErr != nil {
+				return s.publicNode(updated), clearErr
+			}
+		}
 	}
 	return s.publicNode(updated), err
 }
@@ -503,6 +642,9 @@ func (s *Service) UpdateProxyProfile(ctx context.Context, id uint64, input Proxy
 	}
 	if proxyChanged {
 		s.forgetClearances(nodeIDs)
+		if err := s.clearQualityLeasesForNodes(ctx, nodeIDs); err != nil {
+			return s.publicProxyProfile(updated), err
+		}
 	}
 	return s.publicProxyProfile(updated), nil
 }
@@ -695,6 +837,11 @@ func (s *Service) UpdateManyEnabled(ctx context.Context, nodeIDs []uint64, enabl
 		}
 		if updated > 0 {
 			s.forgetClearances(ids)
+			if !enabled {
+				if clearErr := s.clearQualityLeasesForNodes(ctx, ids); clearErr != nil {
+					return updated, clearErr
+				}
+			}
 		}
 		return updated, nil
 	}
@@ -718,9 +865,22 @@ func (s *Service) UpdateManyEnabled(ctx context.Context, nodeIDs []uint64, enabl
 		s.forgetClearance(id)
 		updated++
 	}
+	if !enabled && updated > 0 {
+		if err := s.clearQualityLeasesForNodes(ctx, ids); err != nil {
+			return updated, err
+		}
+	}
 	return updated, nil
 }
 
+func (s *Service) clearQualityLeasesForNodes(ctx context.Context, nodeIDs []uint64) error {
+	if s.qualityLeases == nil || len(nodeIDs) == 0 {
+		return nil
+	}
+	_, err := s.qualityLeases.DeleteEgressLeaseBlocksByNodes(ctx, uniqueIDs(nodeIDs))
+	return err
+}
+
 func (s *Service) Delete(ctx context.Context, id uint64) error {
 	err := s.repository.DeleteEgressNode(ctx, id)
 	if errors.Is(err, repository.ErrNotFound) {

backend/internal/application/gateway/quality_probe.go

@@ -78,7 +78,7 @@ func (s *Service) ProbeEgressQuality(ctx context.Context, nodeID uint64, input e
 	probeCtx := infraegress.WithQualityProbe(ctx)
 	result, err := s.CreateChatCompletion(probeCtx, Input{
 		RequestID: requestID, ClientKey: key, PublicModel: publicModel, Body: body,
-		Streaming: true, Operation: audit.OperationChat, ForcedEgressNodeID: nodeID,
+		Streaming: true, Operation: audit.OperationChat, ForcedEgressNodeID: nodeID, ForcedAccountID: input.AccountID,
 	})
 	if err != nil {
 		return egressapp.QualityProbeResult{}, normalizeQualityProbeRequestError(err)

backend/internal/application/gateway/selector.go

@@ -491,6 +491,11 @@ func (s *Selector) acquire(ctx context.Context, provider account.Provider, model
 			earliestRetry = earlierFuture(earliestRetry, candidate.ModelQuotaBlock.CooldownUntil, now)
 			continue
 		}
+		if candidateEgressLeaseCooling(candidate, value, now) {
+			coolingCandidates++
+			earliestRetry = earlierFuture(earliestRetry, candidate.EgressLeaseBlock.CooldownUntil, now)
+			continue
+		}
 		if value.CooldownUntil != nil && now.Before(*value.CooldownUntil) {
 			coolingCandidates++
 			earliestRetry = earlierFuture(earliestRetry, *value.CooldownUntil, now)
@@ -752,14 +757,20 @@ func isSelectionUnavailable(err error, reason SelectionUnavailableReason) bool {
 
 // AcquirePinned 为 previous_response_id 等账号归属请求获取指定账号租约。
 func (s *Selector) AcquirePinned(ctx context.Context, provider account.Provider, accountID, modelRouteID uint64, upstreamModel, quotaMode string, inference bool) (*accountLease, error) {
-	return s.acquirePinned(ctx, provider, accountID, modelRouteID, upstreamModel, quotaMode, inference, clientkeydomain.AccountScope{})
+	return s.acquirePinned(ctx, provider, accountID, modelRouteID, upstreamModel, quotaMode, inference, false, clientkeydomain.AccountScope{})
 }
 
 func (s *Selector) AcquirePinnedForKey(ctx context.Context, provider account.Provider, accountID, modelRouteID uint64, upstreamModel, quotaMode string, inference bool, scope clientkeydomain.AccountScope) (*accountLease, error) {
-	return s.acquirePinned(ctx, provider, accountID, modelRouteID, upstreamModel, quotaMode, inference, scope)
+	return s.acquirePinned(ctx, provider, accountID, modelRouteID, upstreamModel, quotaMode, inference, false, scope)
+}
+
+// AcquirePinnedForQualityProbe keeps every ordinary eligibility check while
+// bypassing only the exact account+node lease block being recovery-tested.
+func (s *Selector) AcquirePinnedForQualityProbe(ctx context.Context, provider account.Provider, accountID, modelRouteID uint64, upstreamModel, quotaMode string, scope clientkeydomain.AccountScope) (*accountLease, error) {
+	return s.acquirePinned(ctx, provider, accountID, modelRouteID, upstreamModel, quotaMode, true, true, scope)
 }
 
-func (s *Selector) acquirePinned(ctx context.Context, provider account.Provider, accountID, modelRouteID uint64, upstreamModel, quotaMode string, inference bool, requestedScope clientkeydomain.AccountScope) (lease *accountLease, err error) {
+func (s *Selector) acquirePinned(ctx context.Context, provider account.Provider, accountID, modelRouteID uint64, upstreamModel, quotaMode string, inference, ignoreEgressLeaseBlock bool, requestedScope clientkeydomain.AccountScope) (lease *accountLease, err error) {
 	accountScope, scopeValid := clientkeydomain.NormalizeAccountScope(requestedScope)
 	defer annotateSelectionAccountScope(&err, accountScope)
 	if !scopeValid || !accountScope.AllowsProvider(provider) {
@@ -790,6 +801,9 @@ func (s *Selector) acquirePinned(ctx context.Context, provider account.Provider,
 			if candidate.ModelQuotaBlock != nil && now.Before(candidate.ModelQuotaBlock.CooldownUntil) {
 				return nil, &SelectionUnavailableError{Reason: SelectionModelCooling, RetryAfter: retryDelay(now, candidate.ModelQuotaBlock.CooldownUntil)}
 			}
+			if !ignoreEgressLeaseBlock && candidateEgressLeaseCooling(candidate, value, now) {
+				return nil, &SelectionUnavailableError{Reason: SelectionCooling, RetryAfter: retryDelay(now, candidate.EgressLeaseBlock.CooldownUntil)}
+			}
 			if value.CooldownUntil != nil && now.Before(*value.CooldownUntil) {
 				return nil, &SelectionUnavailableError{Reason: SelectionCooling, RetryAfter: retryDelay(now, *value.CooldownUntil)}
 			}
@@ -901,6 +915,11 @@ func effectiveQuotaMode(candidate account.RoutingCandidate, fallback string) str
 	return fallback
 }
 
+func candidateEgressLeaseCooling(candidate account.RoutingCandidate, credential account.Credential, now time.Time) bool {
+	block := candidate.EgressLeaseBlock
+	return block != nil && block.AccountID == credential.ID && block.NodeID != 0 && credential.EgressNodeID == block.NodeID && now.Before(block.CooldownUntil)
+}
+
 // candidateSupportsModel treats a recognized Web catalog entry as an
 // effective capability for tiers that the adapter explicitly allows. This
 // prevents a historical capability snapshot from blocking a newly enabled
@@ -1889,7 +1908,7 @@ func assembleRoutingCandidates(provider account.Provider, quotaMode string, base
 		}
 		result = append(result, account.RoutingCandidate{
 			Credential: base.Credential, Billing: base.Billing, QuotaWindow: base.QuotaWindow, QuotaRecovery: base.QuotaRecovery,
-			ModelQuotaBlock: overlayValue.ModelQuotaBlock, ModelCapabilityKnown: known, SupportsModel: supports,
+			EgressLeaseBlock: base.EgressLeaseBlock, ModelQuotaBlock: overlayValue.ModelQuotaBlock, ModelCapabilityKnown: known, SupportsModel: supports,
 		})
 	}
 	return result

backend/internal/application/gateway/selector_session.go

@@ -84,6 +84,11 @@ func (s *Selector) beginSelectionSessionForKey(ctx context.Context, provider acc
 			earliestRetry = earlierFuture(earliestRetry, candidate.ModelQuotaBlock.CooldownUntil, now)
 			continue
 		}
+		if candidateEgressLeaseCooling(candidate, value, now) {
+			coolingCandidates++
+			earliestRetry = earlierFuture(earliestRetry, candidate.EgressLeaseBlock.CooldownUntil, now)
+			continue
+		}
 		if value.CooldownUntil != nil && now.Before(*value.CooldownUntil) {
 			coolingCandidates++
 			earliestRetry = earlierFuture(earliestRetry, *value.CooldownUntil, now)

backend/internal/application/gateway/selector_test.go

@@ -141,10 +141,33 @@ func TestSelectorQualityProbePinsAccountToRequestedEgressNode(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer lease.Release()
 	if lease.Credential.ID != second.ID || lease.Credential.ID == first.ID {
 		t.Fatalf("selected account=%d, want=%d on node=%d", lease.Credential.ID, second.ID, secondNode.ID)
 	}
+	lease.Release()
+
+	if _, err := accounts.UpsertEgressLeaseBlock(ctx, account.EgressLeaseBlock{
+		AccountID: second.ID, NodeID: secondNode.ID, Reason: "hard_tps", Version: "selector-lease-0001", CooldownUntil: time.Now().UTC().Add(time.Hour),
+	}); err != nil {
+		t.Fatal(err)
+	}
+	selector = NewSelector(accounts, memory.NewConcurrencyLimiter(), memory.NewStickyStore(), nil, time.Hour, time.Second, time.Minute)
+	if _, err := selector.AcquirePinnedForKey(ctx, account.ProviderBuild, second.ID, 0, "grok-test", "", true, clientkeydomain.AccountScope{}); err == nil {
+		t.Fatal("ordinary pinned inference ignored the active egress lease block")
+	} else {
+		var unavailable *SelectionUnavailableError
+		if !errors.As(err, &unavailable) || unavailable.Reason != SelectionCooling {
+			t.Fatalf("ordinary pinned error = %v", err)
+		}
+	}
+	recoveryLease, err := selector.AcquirePinnedForQualityProbe(ctx, account.ProviderBuild, second.ID, 0, "grok-test", "", clientkeydomain.AccountScope{})
+	if err != nil {
+		t.Fatalf("quality recovery did not bypass only the lease block: %v", err)
+	}
+	defer recoveryLease.Release()
+	if recoveryLease.Credential.ID != second.ID || recoveryLease.Credential.EgressNodeID != secondNode.ID {
+		t.Fatalf("quality recovery lease = %#v", recoveryLease.Credential)
+	}
 }
 
 func TestSelectorQualityProbeBorrowsHealthyAccountForUnavailableNode(t *testing.T) {