feat(model): LLM token 预算可配置化 — models.json每模型maxTokens + LLM_FALLBACK_MAX_TOKENS回退兜底
- config.go: LLMConfig.FallbackMaxTokens(LLM_FALLBACK_MAX_TOKENS, 默认8192) - runtimeconfig/models.go: Group.MaxTokens(models.json每模型独立预算, 负值拒绝) - model/fallback.go: Chain.WithTokenBudget + applyBudget; runtime.go buildSnapshot接入(active.MaxTokens, env.FallbackMaxTokens) - 优先级: 显式Request.MaxTokens > 组maxTokens > 仅多步链末端的LLM_FALLBACK_MAX_TOKENS > 适配器默认8192; 单步链不动env兜底(避免OpenAI主模型行为回归) - gateway admin: AdminLlmProvider.MaxTokens 透传/回显/负值400 - .env.example + .env(LLM_FALLBACK_MAX_TOKENS=8192); Docs/api-client v0.27 + api-reference + design/04 - 新增测试(config/runtimeconfig/budget_test/admin_write), build/vet/test绿+3平台交叉编译
This commit is contained in:
1 parent
f347681034
commit
53002f1e11
13 files changed
+384
-15
No files matched your search
@@ -58,6 +58,10 @@ type AdminLlmProvider struct {
|
||||
Multimodal bool `json:"multimodal"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Fallback string `json:"fallback,omitempty"`
|
||||
// MaxTokens is the group's own token budget (0 = unset; the runtime then
|
||||
// falls back to LLM_FALLBACK_MAX_TOKENS on the last fallback step, then the
|
||||
// adapter default).
|
||||
MaxTokens int `json:"maxTokens,omitempty"`
|
||||
}
|
||||
|
||||
// AdminLlmConfig is the wire shape of GET /api/v1/admin/llm.
|
||||
@@ -360,6 +364,7 @@ func (s *Server) adminGetLlm(w http.ResponseWriter, r *http.Request) {
|
||||
Multimodal: g.Multimodal,
|
||||
Enabled: i == 0,
|
||||
Fallback: g.Fallback,
|
||||
MaxTokens: g.MaxTokens,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, AdminLlmConfig{Providers: providers})
|
||||
|
||||
@@ -37,7 +37,8 @@ const (
|
||||
|
||||
// llmProviderRequest is the POST /api/v1/admin/llm body for adding/updating a
|
||||
// provider group. Name is a client-side display label (not persisted — the
|
||||
// models.json schema carries id/provider/baseUrl/apiKey/model/fallback/multimodal).
|
||||
// models.json schema carries id/provider/baseUrl/apiKey/model/fallback/
|
||||
// multimodal/maxTokens).
|
||||
type llmProviderRequest struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
@@ -47,6 +48,10 @@ type llmProviderRequest struct {
|
||||
Model string `json:"model"`
|
||||
Multimodal bool `json:"multimodal"`
|
||||
Fallback string `json:"fallback"`
|
||||
// MaxTokens is the group's own token budget. Zero means unset (the runtime
|
||||
// falls back to LLM_FALLBACK_MAX_TOKENS on the last fallback step, then the
|
||||
// adapter default). Negative is rejected.
|
||||
MaxTokens int `json:"maxTokens"`
|
||||
}
|
||||
|
||||
// adminAddLlmProvider saves a provider group to models.json (read-modify-write,
|
||||
@@ -64,6 +69,11 @@ func (s *Server) adminAddLlmProvider(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "baseUrl and model are required"))
|
||||
return
|
||||
}
|
||||
if req.MaxTokens < 0 {
|
||||
s.auditClaims(r, "llm.addProvider", "error")
|
||||
writeError(w, apperr.Wrap(apperr.ErrBadRequest, "bad_request", "maxTokens must be >= 0"))
|
||||
return
|
||||
}
|
||||
provider := req.Provider
|
||||
if provider == "" {
|
||||
provider = model.ProviderOpenAI
|
||||
@@ -92,6 +102,7 @@ func (s *Server) adminAddLlmProvider(w http.ResponseWriter, r *http.Request) {
|
||||
Model: req.Model,
|
||||
Fallback: req.Fallback,
|
||||
Multimodal: req.Multimodal,
|
||||
MaxTokens: req.MaxTokens,
|
||||
})
|
||||
if err := runtimeconfig.SaveModels(s.llm.Path(), groups); err != nil {
|
||||
s.auditClaims(r, "llm.addProvider", "error")
|
||||
@@ -114,6 +125,7 @@ func (s *Server) adminAddLlmProvider(w http.ResponseWriter, r *http.Request) {
|
||||
Multimodal: req.Multimodal,
|
||||
Enabled: id == s.llm.Group().ID,
|
||||
Fallback: req.Fallback,
|
||||
MaxTokens: req.MaxTokens,
|
||||
}
|
||||
s.auditClaims(r, "llm.addProvider", "ok")
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
|
||||
@@ -231,7 +231,7 @@ func TestAdminAddLlmProviderPersistsAndReloads(t *testing.T) {
|
||||
tok := signAny(t, jwt, "acct", auth.RoleOwner)
|
||||
|
||||
resp, body := doReq(t, http.MethodPost, ts.URL+"/api/v1/admin/llm",
|
||||
map[string]any{"id": "deepseek", "provider": "openai", "baseUrl": "https://api.deepseek.com/v1", "apiKey": "sk-ds", "model": "deepseek-chat", "multimodal": true}, tok)
|
||||
map[string]any{"id": "deepseek", "provider": "openai", "baseUrl": "https://api.deepseek.com/v1", "apiKey": "sk-ds", "model": "deepseek-chat", "multimodal": true, "maxTokens": 1234}, tok)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("status = %d, want 201 (body=%s)", resp.StatusCode, body)
|
||||
}
|
||||
@@ -243,7 +243,7 @@ func TestAdminAddLlmProviderPersistsAndReloads(t *testing.T) {
|
||||
if err := json.Unmarshal(decodeBody(t, body)["data"], &res); err != nil {
|
||||
t.Fatalf("decode add: %v", err)
|
||||
}
|
||||
if !res.Persisted || res.Path != modelsPath || res.Provider.Model != "deepseek-chat" {
|
||||
if !res.Persisted || res.Path != modelsPath || res.Provider.Model != "deepseek-chat" || res.Provider.MaxTokens != 1234 {
|
||||
t.Fatalf("add res = %+v", res)
|
||||
}
|
||||
|
||||
@@ -252,8 +252,8 @@ func TestAdminAddLlmProviderPersistsAndReloads(t *testing.T) {
|
||||
if warn != nil {
|
||||
t.Fatalf("load models warn = %v", warn)
|
||||
}
|
||||
if len(groups) == 0 || groups[0].ID != "deepseek" || groups[0].APIKey != "sk-ds" {
|
||||
t.Fatalf("file groups = %+v, want deepseek first with sk-ds", groups)
|
||||
if len(groups) == 0 || groups[0].ID != "deepseek" || groups[0].APIKey != "sk-ds" || groups[0].MaxTokens != 1234 {
|
||||
t.Fatalf("file groups = %+v, want deepseek first with sk-ds and maxTokens 1234", groups)
|
||||
}
|
||||
|
||||
// The running stack reflects the new active provider/model (hot reload).
|
||||
@@ -270,8 +270,8 @@ func TestAdminAddLlmProviderPersistsAndReloads(t *testing.T) {
|
||||
if err := json.Unmarshal(decodeBody(t, gBody)["data"], &list); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
if len(list.Providers) == 0 || list.Providers[0].Model != "deepseek-chat" {
|
||||
t.Fatalf("list = %+v, want deepseek-chat first", list.Providers)
|
||||
if len(list.Providers) == 0 || list.Providers[0].Model != "deepseek-chat" || list.Providers[0].MaxTokens != 1234 {
|
||||
t.Fatalf("list = %+v, want deepseek-chat first with maxTokens 1234", list.Providers)
|
||||
}
|
||||
if strings.Contains(string(gBody), "sk-ds") || strings.Contains(string(gBody), "sk-env") {
|
||||
t.Fatalf("API key leaked in GET /admin/llm: %s", gBody)
|
||||
|
||||
Reference in new issue
Block a user