diff --git a/README.md b/README.md index 9a5a1af71..3a07e42cf 100644 --- a/README.md +++ b/README.md @@ -363,17 +363,21 @@ identity automatically: ```yaml qualityGuard: enabled: true - model: "grok-4.5" - # Optional: withhold thinking-model streams that have no reasoning. + model: "grok-4.6" + # Withhold thinking-model streams that have no streamed reasoning. + # Observe for up to 30s. An open stream with a reasoning start and visible + # output is released at the deadline; empty/terminal failures still retry. requestRetry: - enabled: false + enabled: true maxAttempts: 6 - holdTimeout: 3s - minOutputTokens: 32 + holdTimeout: 30s + minOutputTokens: 8 onExhausted: fail_closed # fail_open | fail_closed + accountCooldown: 12h + idleAccountCooldown: 15m ``` -`requestRetry` runs on the gateway request path and is independent of the sidecar. It is off by default. When enabled, a thinking-model stream with enough visible output and no reasoning is **not delivered**; another account is tried. If every attempt still has no reasoning, `onExhausted` either returns `503 quality_degraded` or delivers the last body. Image, video, tool, stored-response, and ForcedEgress probe requests are unchanged. +`requestRetry` runs on the gateway request path and is independent of the sidecar. The example enables it. When enabled, a thinking-model stream with enough visible output and no streamed reasoning is **not delivered**; another account is tried. If every attempt still has no reasoning, `onExhausted` either returns `503 quality_degraded` or delivers the last body. Image, video, stored-response, and ForcedEgress probe requests are unchanged. Grok TUI tool turns stay held so 0-thinking dumps cannot skip the gate. ```bash docker compose --profile quality-guard up -d --build diff --git a/README.zh-CN.md b/README.zh-CN.md index 584b95b16..932c80950 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -359,17 +359,21 @@ Hysteria 与 TUIC 暂未支持。FlareSolverr 仅接受 HTTP/SOCKS 代理地址 ```yaml qualityGuard: enabled: true - model: "grok-4.5" - # 可选：思考模型缺 reasoning 时先扣住响应，换号再打，不把降智正文发给用户。 + model: "grok-4.6" + # 思考模型缺流式 reasoning 时先扣住响应，换号再打，不把降智正文发给用户。 + # 最多观察 30 秒；已有 reasoning 起始信号和可见输出的进行中流会在超时后放行， + # 空流和终态仍无 thinking 的响应继续换号。 requestRetry: - enabled: false + enabled: true maxAttempts: 6 - holdTimeout: 3s - minOutputTokens: 32 + holdTimeout: 30s + minOutputTokens: 8 onExhausted: fail_closed # fail_open | fail_closed + accountCooldown: 12h + idleAccountCooldown: 15m ``` -`requestRetry` 在网关请求路径上生效，与 sidecar 探测/隔离相互独立。默认关闭。开启后，可见输出达到 `minOutputTokens` 且全程无 reasoning 时**不发给用户**，排除该账号再试；全部仍无推理则按 `onExhausted` 返回 `503 quality_degraded` 或放出最后一枪。不处理图/视频/工具、stored response 钉账号和 ForcedEgress 探针。 +`requestRetry` 在网关请求路径上生效，与 sidecar 探测/隔离相互独立。示例配置默认开启。开启后，可见输出达到 `minOutputTokens` 且全程无流式 reasoning 时**不发给用户**，排除该账号再试；全部仍无推理则按 `onExhausted` 返回 `503 quality_degraded` 或放出最后一枪。不处理图/视频、stored response 钉账号和 ForcedEgress 探针。Grok TUI 带 tools 的回合仍会 hold，避免 0-thinking 降智流跳过闸门。 ```bash docker compose --profile quality-guard up -d --build diff --git a/backend/internal/application/gateway/quality_retry.go b/backend/internal/application/gateway/quality_retry.go index 548fca792..3a3869443 100644 --- a/backend/internal/application/gateway/quality_retry.go +++ b/backend/internal/application/gateway/quality_retry.go @@ -21,14 +21,14 @@ const ( qualityRetryFailOpen = "fail_open" qualityRetryFailClosed = "fail_closed" defaultQualityMaxAttempts = 6 - defaultQualityHoldTimeout = 3 * time.Second - defaultQualityMinOutput = int64(32) - defaultMissingThinkingCooldown = 24 * time.Hour + defaultQualityHoldTimeout = 30 * time.Second + defaultQualityMinOutput = int64(8) + defaultMissingThinkingCooldown = 12 * time.Hour lastErrorMissingThinking = accountdomain.LastErrorMissingThinking lastErrorMissingThinkingDisabled = accountdomain.LastErrorMissingThinkingDisabled // An empty stream that idles while held is treated as an account-quality // failure: the request can still rotate before any bytes reach the client. - qualityIdleAccountCooldown = 24 * time.Hour + qualityIdleAccountCooldown = 15 * time.Minute ) var ( @@ -129,9 +129,12 @@ func (s *Service) qualityRetryConfig() QualityRetryRuntime { // A hold timeout with no visible output is not fail-open: keep waiting for // more bytes or a stream abort so an empty hang is not flushed as HTTP 200. // -// An empty reasoning stub is not thinking. Wait for usage/terminal so -// encrypted thinking (tokens arrive at the end) is not withheld, and so -// 200 + 推理·高 + reasoning=0 is not delivered the moment the stub appears. +// An empty reasoning stub is not thinking. Before the hold deadline, wait for +// real evidence or a terminal event. If the deadline expires while the stream +// is still open and already has visible output, the result is inconclusive: +// release it without penalizing the account. A stub-only empty stream keeps +// waiting for idle/terminal handling. This keeps HoldTimeout a real latency +// bound without reopening the empty-stream 200 response path. func ClassifyQualityHold(sig QualityStreamSignals, minOutput int64) QualityVerdict { if minOutput <= 0 { minOutput = defaultQualityMinOutput @@ -148,7 +151,10 @@ func ClassifyQualityHold(sig QualityStreamSignals, minOutput int64) QualityVerdi output = sig.OutputTokens } enough := output >= minOutput - if sig.ReasoningStarted && !sig.Terminal && !sig.HoldExpired { + if sig.ReasoningStarted && !sig.Terminal { + if sig.HoldExpired && output > 0 { + return QualityDeliver + } return QualityWait } if sig.Terminal { diff --git a/backend/internal/application/gateway/quality_retry_test.go b/backend/internal/application/gateway/quality_retry_test.go index 1e619dc5f..e86c8b47e 100644 --- a/backend/internal/application/gateway/quality_retry_test.go +++ b/backend/internal/application/gateway/quality_retry_test.go @@ -41,6 +41,8 @@ func TestClassifyQualityHold(t *testing.T) { {name: "empty terminal waits for transport handling", sig: QualityStreamSignals{Terminal: true}, want: QualityWait}, {name: "midstream enough content withhold", sig: QualityStreamSignals{VisibleTokens: 64}, want: QualityWithhold}, {name: "stub midstream waits even with enough visible", sig: QualityStreamSignals{ReasoningStarted: true, VisibleTokens: 64}, want: QualityWait}, + {name: "stub hold expiry is inconclusive and delivers", sig: QualityStreamSignals{ReasoningStarted: true, VisibleTokens: 64, HoldExpired: true}, want: QualityDeliver}, + {name: "stub-only hold expiry keeps waiting", sig: QualityStreamSignals{ReasoningStarted: true, HoldExpired: true}, want: QualityWait}, {name: "stub terminal enough withhold", sig: QualityStreamSignals{ReasoningStarted: true, VisibleTokens: 64, Terminal: true}, want: QualityWithhold}, {name: "wait for more", sig: QualityStreamSignals{VisibleTokens: 8}, want: QualityWait}, {name: "hold expired short delivers", sig: QualityStreamSignals{VisibleTokens: 8, HoldExpired: true}, want: QualityDeliver}, @@ -606,6 +608,64 @@ func TestPeekQualityStreamHoldTimeoutInterruptsBlockedReadAndPreservesRemainder( } } +func TestPeekQualityStreamHoldTimeoutDeliversStartedReasoningAndPreservesLateEvidence(t *testing.T) { + t.Parallel() + reader, writer := io.Pipe() + content := strings.Repeat("abcd", 40) + first := sse( + `data: {"type":"response.output_item.added","item":{"id":"rs_1","type":"reasoning"}}`, + `data: {"type":"response.output_text.delta","delta":"`+content+`"}`, + ) + second := sse( + `data: {"type":"response.output_item.done","item":{"id":"rs_1","type":"reasoning","encrypted_content":"late-proof"}}`, + `data: {"type":"response.completed","response":{"id":"resp_1","usage":{"output_tokens":40,"output_tokens_details":{"reasoning_tokens":20}}}}`, + ) + writeErr := make(chan error, 1) + continueWrite := make(chan struct{}) + go func() { + if _, err := io.WriteString(writer, first); err != nil { + writeErr <- err + return + } + select { + case <-continueWrite: + case <-time.After(500 * time.Millisecond): + } + if _, err := io.WriteString(writer, second); err != nil { + writeErr <- err + return + } + writeErr <- writer.Close() + }() + + started := time.Now() + replay, verdict, _, _, err := peekQualityStream(context.Background(), reader, qualityProtocolResponses, QualityRetryRuntime{ + MinOutputTokens: 8, + HoldTimeout: 30 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + defer replay.Close() + if elapsed := time.Since(started); elapsed < 20*time.Millisecond || elapsed > 200*time.Millisecond { + t.Fatalf("peek returned after %s, want the 30ms hold timeout", elapsed) + } + if verdict != QualityDeliver { + t.Fatalf("started reasoning at hold timeout verdict = %s, want deliver", verdict) + } + close(continueWrite) + body, err := io.ReadAll(replay) + if err != nil { + t.Fatal(err) + } + if err := <-writeErr; err != nil { + t.Fatal(err) + } + if got := string(body); !strings.Contains(got, content) || !strings.Contains(got, `"encrypted_content":"late-proof"`) { + t.Fatalf("replay lost late reasoning evidence: %q", got) + } +} + func TestPeekQualityStreamHoldTimeoutEmptyDoesNotFailOpen(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancelCause(context.Background()) @@ -640,6 +700,48 @@ func TestPeekQualityStreamHoldTimeoutEmptyDoesNotFailOpen(t *testing.T) { } } +func TestPeekQualityStreamHoldTimeoutStubOnlyDoesNotFailOpen(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancelCause(context.Background()) + reader, writer := io.Pipe() + defer writer.Close() + writeDone := make(chan error, 1) + go func() { + _, err := io.WriteString(writer, sse(": grok2api-reasoning-start")) + writeDone <- err + }() + done := make(chan struct{}) + var verdict QualityVerdict + var peekErr error + go func() { + defer close(done) + _, verdict, _, _, peekErr = peekQualityStream(ctx, reader, qualityProtocolChat, QualityRetryRuntime{ + MinOutputTokens: 8, + HoldTimeout: 20 * time.Millisecond, + }) + }() + if err := <-writeDone; err != nil { + t.Fatal(err) + } + select { + case <-done: + t.Fatal("stub-only hold timeout must keep reading, not release an empty stream") + case <-time.After(50 * time.Millisecond): + } + cancel(neterrorpkg.ErrUpstreamStreamIdleTimeout) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("peekQualityStream did not return after stub-only idle cancel") + } + if !neterrorpkg.IsUpstreamStreamIdleTimeout(peekErr) { + t.Fatalf("peekErr = %v, want idle timeout", peekErr) + } + if verdict != QualityWait { + t.Fatalf("verdict=%s, want wait so the loop retries as transport", verdict) + } +} + type qualityOpenPeekResult struct { replay io.ReadCloser verdict QualityVerdict @@ -1089,8 +1191,8 @@ func TestAttemptLoopQualityHold(t *testing.T) { if emptyAccount.FailureCount != 1 || emptyAccount.CooldownUntil == nil { t.Fatalf("empty stream account was not cooled: %#v", emptyAccount) } - if remaining := time.Until(*emptyAccount.CooldownUntil); remaining < 23*time.Hour || remaining > 24*time.Hour+time.Minute { - t.Fatalf("empty stream cooldown = %s, want about 24h", remaining) + if remaining := time.Until(*emptyAccount.CooldownUntil); remaining < 14*time.Minute || remaining > 15*time.Minute+time.Minute { + t.Fatalf("empty stream cooldown = %s, want about 15m", remaining) } noThinkingAccount, err := accountRepo.Get(ctx, credentials[1].ID) if err != nil { @@ -1099,8 +1201,8 @@ func TestAttemptLoopQualityHold(t *testing.T) { if !noThinkingAccount.Enabled || noThinkingAccount.LastError != lastErrorMissingThinking || noThinkingAccount.CooldownUntil == nil { t.Fatalf("missing-thinking account was not cooled: %#v", noThinkingAccount) } - if remaining := time.Until(*noThinkingAccount.CooldownUntil); remaining < 23*time.Hour || remaining > 24*time.Hour+time.Minute { - t.Fatalf("missing-thinking cooldown = %s, want about 24h", remaining) + if remaining := time.Until(*noThinkingAccount.CooldownUntil); remaining < 11*time.Hour || remaining > 12*time.Hour+time.Minute { + t.Fatalf("missing-thinking cooldown = %s, want about 12h", remaining) } logs, total, err := auditRepo.List(ctx, 0, 20) if err != nil { @@ -1307,7 +1409,7 @@ func TestAttemptLoopQualityFailOpenFallbackAndTotalAttemptCap(t *testing.T) { func TestNormalizeQualityRetryDefaults(t *testing.T) { t.Parallel() got := normalizeQualityRetry(QualityRetryRuntime{Enabled: true}) - if !got.Enabled || got.MaxAttempts != 6 || got.MinOutputTokens != 32 || got.OnExhausted != qualityRetryFailClosed || got.HoldTimeout != 3*time.Second || got.AccountCooldown != 24*time.Hour || got.IdleAccountCooldown != 24*time.Hour { + if !got.Enabled || got.MaxAttempts != 6 || got.MinOutputTokens != 8 || got.OnExhausted != qualityRetryFailClosed || got.HoldTimeout != 30*time.Second || got.AccountCooldown != 12*time.Hour || got.IdleAccountCooldown != 15*time.Minute { t.Fatalf("defaults = %#v", got) } } diff --git a/backend/internal/application/gateway/service_test.go b/backend/internal/application/gateway/service_test.go index 9e749ef8c..95350dad5 100644 --- a/backend/internal/application/gateway/service_test.go +++ b/backend/internal/application/gateway/service_test.go @@ -443,8 +443,8 @@ func TestGatewayFailsOverBeforeReturningBody(t *testing.T) { t.Fatalf("interrupted account health = %#v, err=%v", interruptedAccount, err) } remaining := time.Until(*interruptedAccount.CooldownUntil) - if remaining < 23*time.Hour || remaining > 24*time.Hour+time.Minute { - t.Fatalf("idle stream cooldown = %s, want about 24h", remaining) + if remaining < 14*time.Minute || remaining > 15*time.Minute+time.Minute { + t.Fatalf("idle stream cooldown = %s, want about 15m", remaining) } } diff --git a/backend/internal/infra/config/config.go b/backend/internal/infra/config/config.go index 3cfa79d82..28541940d 100644 --- a/backend/internal/infra/config/config.go +++ b/backend/internal/infra/config/config.go @@ -296,7 +296,7 @@ type QualityGuardRequestRetryConfig struct { OnExhausted string `yaml:"onExhausted"` AccountCooldown Duration `yaml:"accountCooldown"` // IdleAccountCooldown cools an account after a truly empty upstream - // stream. Independent of accountCooldown (missing-thinking). Zero uses 24h. + // stream. Independent of accountCooldown (missing-thinking). Zero uses 15m. IdleAccountCooldown Duration `yaml:"idleAccountCooldown"` } @@ -935,15 +935,15 @@ func defaultConfig() Config { LedgerUnhealthyGrace: Duration(10 * time.Second), LedgerQueueHighWatermarkPct: 90, }, QualityGuard: QualityGuardConfig{ - Model: "grok-4.5", Mode: "hybrid", + Model: "grok-4.6", Mode: "hybrid", ActiveInterval: Duration(30 * time.Minute), PassivePollInterval: Duration(5 * time.Second), SoftTPS: 500, HardTPS: 1000, ConsecutiveSoft: 2, ConsecutiveErrors: 2, QuarantineDuration: Duration(5 * time.Minute), NoAccountBackoff: Duration(5 * time.Minute), MinimumHealthyNodes: 3, MaxOutputTokens: 384, MinimumGenerationWindow: Duration(time.Second), RotationTimeout: Duration(45 * time.Second), RequestRetry: QualityGuardRequestRetryConfig{ - MaxAttempts: 6, HoldTimeout: Duration(3 * time.Second), MinOutputTokens: 32, OnExhausted: "fail_closed", - AccountCooldown: Duration(24 * time.Hour), IdleAccountCooldown: Duration(24 * time.Hour), + MaxAttempts: 6, HoldTimeout: Duration(30 * time.Second), MinOutputTokens: 8, OnExhausted: "fail_closed", + AccountCooldown: Duration(12 * time.Hour), IdleAccountCooldown: Duration(15 * time.Minute), }, }, ClientKeyDefaults: ClientKeyDefaultsConfig{RPMLimit: clientkeydomain.DefaultRPMLimit, MaxConcurrent: clientkeydomain.DefaultMaxConcurrent}, diff --git a/backend/internal/infra/config/config_test.go b/backend/internal/infra/config/config_test.go index c131d2153..fb12df85b 100644 --- a/backend/internal/infra/config/config_test.go +++ b/backend/internal/infra/config/config_test.go @@ -215,7 +215,7 @@ qualityGuard: t.Fatalf("qualityGuard = %#v", value.QualityGuard) } retry := value.QualityGuard.RequestRetry - if retry.Enabled || retry.MaxAttempts != 6 || retry.HoldTimeout.Value() != 3*time.Second || retry.MinOutputTokens != 32 || retry.OnExhausted != "fail_closed" || retry.AccountCooldown.Value() != 24*time.Hour { + if retry.Enabled || retry.MaxAttempts != 6 || retry.HoldTimeout.Value() != 30*time.Second || retry.MinOutputTokens != 8 || retry.OnExhausted != "fail_closed" || retry.AccountCooldown.Value() != 12*time.Hour { t.Fatalf("loaded requestRetry defaults = %#v", retry) } } @@ -223,7 +223,7 @@ qualityGuard: func TestDefaultQualityGuardRequestRetryContract(t *testing.T) { t.Parallel() got := defaultConfig().QualityGuard.RequestRetry - if got.Enabled || got.MaxAttempts != 6 || got.HoldTimeout.Value() != 3*time.Second || got.MinOutputTokens != 32 || got.OnExhausted != "fail_closed" || got.AccountCooldown.Value() != 24*time.Hour || got.IdleAccountCooldown.Value() != 24*time.Hour { + if got.Enabled || got.MaxAttempts != 6 || got.HoldTimeout.Value() != 30*time.Second || got.MinOutputTokens != 8 || got.OnExhausted != "fail_closed" || got.AccountCooldown.Value() != 12*time.Hour || got.IdleAccountCooldown.Value() != 15*time.Minute { t.Fatalf("requestRetry defaults = %#v", got) } } diff --git a/config.example.yaml b/config.example.yaml index 37db00d46..ecc9cc0ed 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -129,7 +129,7 @@ qualityGuard: enabled: false # 主程序会自动创建并复用不可导出的内部探测身份，无需配置 Client Key。 # 使用已验证会输出推理 Token 的 Build 模型时，内置恢复探针还会校验 reasoning Token。 - model: "grok-4.5" + model: "grok-4.6" mode: hybrid # passive | active | hybrid activeInterval: 30m passivePollInterval: 5s @@ -149,19 +149,25 @@ qualityGuard: rotationTimeout: 45s nodeIDs: [] rotatableNodeIDs: [] - # Optional request-path withhold/retry for thinking models. Disabled by default. - # Missing thinking with enough visible output is not delivered; another account - # is tried (up to maxAttempts). fail_closed returns 503 instead of the last body. + # Optional request-path withhold/retry for thinking models. + # Observe up to 30s for real reasoning evidence. If upstream has announced + # a reasoning item and visible output but proof arrives later, the still-open + # stream is released at the deadline without penalizing the account. Empty or + # terminal no-thinking responses are still retried. + # Missing thinking with enough visible output is not delivered; another + # account is tried (up to maxAttempts). fail_closed returns 503 instead + # of the last body. Sidecar qualityGuard.enabled stays false until you + # start the quality-guard profile. requestRetry: - enabled: false + enabled: true maxAttempts: 6 - holdTimeout: 3s - minOutputTokens: 32 + holdTimeout: 30s + minOutputTokens: 8 onExhausted: fail_closed # fail_open | fail_closed # First missing-thinking hit cools the account; a later hit after the # cooldown expires disables it. - accountCooldown: 24h + accountCooldown: 12h # Empty SSE / idle-timeout penalty. Independent of accountCooldown. # Single-account pools should set this lower; toggling enabled does not # clear cooldown — use POST /api/admin/v1/accounts/:id/clear-cooldown. - idleAccountCooldown: 24h + idleAccountCooldown: 15m diff --git a/tools/egress-quality-guard/README.md b/tools/egress-quality-guard/README.md index 7c5d254b5..72b6c2ace 100644 --- a/tools/egress-quality-guard/README.md +++ b/tools/egress-quality-guard/README.md @@ -14,7 +14,7 @@ your own traffic before allowing automatic quarantine. - Supports Grok Build streaming requests after egress nodes and request audits are configured in grok2api. - At least one schedulable Grok Build account must be able to serve the probe model. The account does not have to be bound to every managed node. -- The built-in thinking guard is enforced only when the backend recognizes the configured Build model as reasoning-capable. Keep the default `grok-4.5` or another verified reasoning model when missing-thinking detection is required; unknown and non-reasoning models retain marker/TPS checks without this signal. +- The built-in thinking guard is enforced only when the backend recognizes the configured Build model as reasoning-capable. Keep the default `grok-4.6` or another verified reasoning model when missing-thinking detection is required; unknown and non-reasoning models retain marker/TPS checks without this signal. - The main service automatically provisions a non-exportable system probe identity. The sidecar reaches only a scoped internal API over the Compose network. - Classification is heuristic evidence. It cannot prove that upstream model capability changed and does not replace application-level regression tests. @@ -147,7 +147,7 @@ copy, select, or configure a Client Key for the guard: ```yaml qualityGuard: enabled: true - model: "grok-4.5" + model: "grok-4.6" mode: hybrid activeInterval: 30m passivePollInterval: 5s diff --git a/tools/egress-quality-guard/README.zh-CN.md b/tools/egress-quality-guard/README.zh-CN.md index 2e8d2e239..37a751768 100644 --- a/tools/egress-quality-guard/README.zh-CN.md +++ b/tools/egress-quality-guard/README.zh-CN.md @@ -94,7 +94,7 @@ Webhook，确认出口发生变化，再执行一次真实模型质量检测； ```yaml qualityGuard: enabled: true - model: "grok-4.5" + model: "grok-4.6" mode: hybrid activeInterval: 30m passivePollInterval: 5s