feat(model): LLM token 预算可配置化 — models.json每模型maxTokens + LLM_FALLBACK_MAX_TOKENS回退兜底
- config.go: LLMConfig.FallbackMaxTokens(LLM_FALLBACK_MAX_TOKENS, 默认8192) - runtimeconfig/models.go: Group.MaxTokens(models.json每模型独立预算, 负值拒绝) - model/fallback.go: Chain.WithTokenBudget + applyBudget; runtime.go buildSnapshot接入(active.MaxTokens, env.FallbackMaxTokens) - 优先级: 显式Request.MaxTokens > 组maxTokens > 仅多步链末端的LLM_FALLBACK_MAX_TOKENS > 适配器默认8192; 单步链不动env兜底(避免OpenAI主模型行为回归) - gateway admin: AdminLlmProvider.MaxTokens 透传/回显/负值400 - .env.example + .env(LLM_FALLBACK_MAX_TOKENS=8192); Docs/api-client v0.27 + api-reference + design/04 - 新增测试(config/runtimeconfig/budget_test/admin_write), build/vet/test绿+3平台交叉编译
This commit is contained in:
1 parent
f347681034
commit
53002f1e11
13 files changed
+384
-15
No files matched your search
@@ -90,6 +90,13 @@ type LLMConfig struct {
|
||||
// set, the model router routes image content to it (see model.Router.WithVisionModel).
|
||||
VisionModel string `env:"LLM_VISION_MODEL"`
|
||||
|
||||
// FallbackMaxTokens caps the last-resort fallback model in a group's chain
|
||||
// (primary → fallback). It is the final safety budget: it only applies when
|
||||
// the chain actually has a fallback step, that group sets no per-group
|
||||
// MaxTokens (models.json maxTokens) and the request carries no explicit
|
||||
// budget. Zero disables the cap (the adapter default then applies).
|
||||
FallbackMaxTokens int `env:"LLM_FALLBACK_MAX_TOKENS" default:"8192"`
|
||||
|
||||
// Anthropic native provider, used when Provider=anthropic. Independent
|
||||
// config block so each vendor is a self-contained config.
|
||||
AnthropicBaseURL string `env:"ANTHROPIC_BASE_URL"`
|
||||
|
||||
@@ -91,6 +91,55 @@ IOT_PROVIDER=none
|
||||
}
|
||||
}
|
||||
|
||||
// unsetEnvForTest clears key for the test and restores the original value (or
|
||||
// absence) on cleanup, so a .env file is not shadowed by an ambient process env
|
||||
// variable. config tests are not parallel, so this is safe.
|
||||
func unsetEnvForTest(t *testing.T, key string) {
|
||||
t.Helper()
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
t.Cleanup(func() { _ = os.Setenv(key, v) })
|
||||
} else {
|
||||
t.Cleanup(func() { _ = os.Unsetenv(key) })
|
||||
}
|
||||
if err := os.Unsetenv(key); err != nil {
|
||||
t.Fatalf("unset %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_LlmFallbackMaxTokens(t *testing.T) {
|
||||
t.Setenv("JWT_SECRET", strings.Repeat("s", 40))
|
||||
unsetEnvForTest(t, "LLM_FALLBACK_MAX_TOKENS")
|
||||
|
||||
// Default: the last-resort fallback budget is a meaningful 8192.
|
||||
cfg, err := Load("")
|
||||
if err != nil {
|
||||
t.Fatalf("Load() error: %v", err)
|
||||
}
|
||||
if cfg.LLM.FallbackMaxTokens != 8192 {
|
||||
t.Errorf("LLM.FallbackMaxTokens = %d, want default 8192", cfg.LLM.FallbackMaxTokens)
|
||||
}
|
||||
|
||||
// A .env file is parsed into the field.
|
||||
path := writeEnvFile(t, "LLM_FALLBACK_MAX_TOKENS=4096\n")
|
||||
cfg, err = Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(%q) error: %v", path, err)
|
||||
}
|
||||
if cfg.LLM.FallbackMaxTokens != 4096 {
|
||||
t.Errorf("LLM.FallbackMaxTokens = %d, want 4096 from .env", cfg.LLM.FallbackMaxTokens)
|
||||
}
|
||||
|
||||
// Process env wins over .env.
|
||||
t.Setenv("LLM_FALLBACK_MAX_TOKENS", "2048")
|
||||
cfg, err = Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load(%q) error: %v", path, err)
|
||||
}
|
||||
if cfg.LLM.FallbackMaxTokens != 2048 {
|
||||
t.Errorf("LLM.FallbackMaxTokens = %d, want 2048 (process env wins)", cfg.LLM.FallbackMaxTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_MissingRequiredFails(t *testing.T) {
|
||||
// Do not set JWT_SECRET.
|
||||
if _, err := Load(""); err == nil {
|
||||
|
||||
@@ -42,6 +42,11 @@ type Group struct {
|
||||
Model string `json:"model"`
|
||||
Fallback string `json:"fallback,omitempty"`
|
||||
Multimodal bool `json:"multimodal,omitempty"`
|
||||
// MaxTokens is the group's own token budget, applied to every model in the
|
||||
// group's fallback chain (primary and fallback). Zero means "unset": the
|
||||
// last-resort fallback step then falls back to LLM_FALLBACK_MAX_TOKENS, and
|
||||
// beyond that the adapter default applies.
|
||||
MaxTokens int `json:"maxTokens,omitempty"`
|
||||
}
|
||||
|
||||
// ErrInvalidGroup is returned when a group fails schema validation.
|
||||
@@ -67,6 +72,9 @@ func (g Group) Validate() error {
|
||||
if strings.TrimSpace(g.Model) == "" {
|
||||
return fmt.Errorf("model is required")
|
||||
}
|
||||
if g.MaxTokens < 0 {
|
||||
return fmt.Errorf("maxTokens must be >= 0, got %d", g.MaxTokens)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -164,6 +164,59 @@ func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGroupMaxTokensRoundTrip(t *testing.T) {
|
||||
// maxTokens is decoded from a hand-written models.json.
|
||||
path := writeModelsFile(t, `[
|
||||
{"id":"openai","provider":"openai","baseUrl":"https://a","apiKey":"sk-1","model":"gpt-4o","maxTokens":4096}
|
||||
]`)
|
||||
groups, warn := LoadModels(path, envCfg())
|
||||
if warn != nil {
|
||||
t.Fatalf("warn = %v, want nil", warn)
|
||||
}
|
||||
if len(groups) != 1 || groups[0].MaxTokens != 4096 {
|
||||
t.Fatalf("groups = %+v, want one group with maxTokens 4096", groups)
|
||||
}
|
||||
|
||||
// SaveModels writes it back, and LoadModels reads it again.
|
||||
out := filepath.Join(t.TempDir(), "models.json")
|
||||
in := []Group{
|
||||
{ID: "openai", Provider: ProviderOpenAI, BaseURL: "https://a", APIKey: "sk-1", Model: "gpt-4o", MaxTokens: 2048},
|
||||
{ID: "claude", Provider: ProviderAnthropic, BaseURL: "https://b", APIKey: "sk-2", Model: "claude"},
|
||||
}
|
||||
if err := SaveModels(out, in); err != nil {
|
||||
t.Fatalf("SaveModels: %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
t.Fatalf("read saved file: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"maxTokens": 2048`) {
|
||||
t.Errorf("saved file missing maxTokens: %s", raw)
|
||||
}
|
||||
round, warn := LoadModels(out, envCfg())
|
||||
if warn != nil {
|
||||
t.Fatalf("load warn = %v", warn)
|
||||
}
|
||||
if round[0].MaxTokens != 2048 || round[1].MaxTokens != 0 {
|
||||
t.Fatalf("round trip maxTokens = (%d,%d), want (2048,0)", round[0].MaxTokens, round[1].MaxTokens)
|
||||
}
|
||||
|
||||
// Upsert and Activate preserve the field.
|
||||
upserted := Upsert(round, Group{ID: "openai", Provider: ProviderOpenAI, BaseURL: "https://a2", Model: "gpt-4o", MaxTokens: 3000})
|
||||
if upserted[0].MaxTokens != 3000 {
|
||||
t.Errorf("Upsert maxTokens = %d, want 3000", upserted[0].MaxTokens)
|
||||
}
|
||||
activated := Activate(round, Group{ID: "claude", Provider: ProviderAnthropic, BaseURL: "https://b", Model: "claude", MaxTokens: 512})
|
||||
if activated[0].MaxTokens != 512 {
|
||||
t.Errorf("Activate maxTokens = %d, want 512", activated[0].MaxTokens)
|
||||
}
|
||||
|
||||
// A negative budget is rejected by validation (never silently persisted).
|
||||
if err := (Group{ID: "x", Provider: ProviderOpenAI, BaseURL: "https://x", Model: "m", MaxTokens: -1}).Validate(); err == nil {
|
||||
t.Fatal("expected Validate to reject a negative maxTokens")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveModels_RejectsInvalid(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "models.json")
|
||||
if err := SaveModels(path, []Group{{ID: "", Provider: "x", BaseURL: "https://a", Model: "m"}}); err == nil {
|
||||
|
||||
Reference in new issue
Block a user