// Package knowledge implements the RAG knowledge-base data source described in // DevDocs/design/10-后台管理面板.md §3.12 (module 8 知识库 RAG) and the // knowledge/memory separation in DevDocs/design/03-记忆.md §3 (③ 知识库 = // 参考资料, 记忆 = 经历). // // It reads the project index (项目索引 → 项目描述 → 章节内容) as the set of knowledge // documents and answers searches over it with line-level context. When a semantic // retriever is wired, it also folds in project-linked long-term memories as // semantic hits, ranked by the retriever's real keyword × semantic × decay path — // so a configured vector index (pgvector / SQLite vector) is genuinely used and // never a shell. // // The store persists knowledge documents by indexing them into the project index // (项目索引 → 项目描述 → 章节内容): IndexDocument chunks a document's content and // stores it as a project plus its chapters, making it text/keyword-searchable // immediately. Semantic/vector embedding of an imported document is a graceful // no-op here (this data source holds no vector index); the existing semantic path // still ranks project-linked memories via the retriever. Deletion remains // unimplemented (M10) and returns ErrWriteUnsupported so the admin handler // answers honestly instead of fabricating a success. package knowledge import ( "context" "crypto/rand" "encoding/hex" "errors" "fmt" "io" "log/slog" "strings" "unicode/utf8" "git.yeij.top/AskaEth/Cyrene_v2/internal/logger" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory" "git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/projectindex" ) // ErrWriteUnsupported is returned by DeleteDocument. Deletion is milestone M10; // the admin handler maps it to a clear 501 so the frontend can explain rather // than show a placeholder alert. var ErrWriteUnsupported = errors.New("knowledge: delete is not implemented in this milestone") // defaultTopK bounds the semantic retrieval path. const defaultTopK = 8 // Retriever is the subset of the memory retriever the knowledge store uses for // semantic ranking. *memory.Retriever implements it; nil means no semantic path. type Retriever interface { Retrieve(ctx context.Context, personaID, query string, topK int) ([]memory.Result, error) } // Doc is one knowledge-base document backing a project-index entry. type Doc struct { // ID is the project index id. ID string // Name is the project title (the document's display name). Name string // Desc is the project description (layer 2 of the hierarchy). Desc string // Chunks is the number of chapters the document is split into. Chunks int // Embedded reports whether a persistent vector index is wired for this // deployment. This is the deployment-level semantic capability; per-document // embedding state is not tracked yet. Embedded bool // Content is the document body supplied to IndexDocument. It is empty for // documents returned by ListDocuments (they carry metadata only) and is // chunked into chapters by IndexDocument. Content string } // Hit is one retrieval result surfaced to the admin panel. type Hit struct { // DocID is the owning project index id. DocID string // DocName is the owning project title. DocName string // Chapter is the chapter title that matched (empty for a semantic/memory hit). Chapter string // Text is the matched content: a chapter's located line(s) with context, or // the memory content for a semantic hit. Text string // Score is the relevance score (higher = more relevant). Score float64 // Semantic reports whether the hit was ranked by the semantic/vector path. Semantic bool } // Deps carries the construction inputs. Every field is nil-tolerant so a store // without a project index or retriever degrades to read-only empty answers. type Deps struct { // Projects is the project index backing the document list and text search. // Optional; when nil the document list and text search return empty. Projects *projectindex.Store // Retriever ranks project-linked memories semantically. Optional; the // semantic path runs only when both Retriever and DefaultPersona are set. Retriever Retriever // DefaultPersona scopes semantic retrieval to one persona. Optional. DefaultPersona string // VectorOn reports whether a persistent vector index (pgvector / SQLite // vector) is wired, so Doc.Embedded is honest and never a shell. VectorOn bool // Log is the base logger. Nil uses a discard logger tagged with this package. Log *slog.Logger } // Store is the RAG data source. It is stateless and safe for concurrent use. type Store struct { projects *projectindex.Store retriever Retriever defaultPersona string vectorOn bool log *slog.Logger } // New builds a Store. Availability degrades gracefully: a nil project index means // no documents, a nil retriever means keyword/line search only. func New(d Deps) *Store { log := d.Log if log == nil { log = slog.New(slog.NewTextHandler(io.Discard, nil)) } return &Store{ projects: d.Projects, retriever: d.Retriever, defaultPersona: strings.TrimSpace(d.DefaultPersona), vectorOn: d.VectorOn, log: logger.WithSource(log, "runtime/knowledge"), } } // ListDocuments enumerates the knowledge-base documents (project-index entries). // A nil project index returns an empty, non-nil slice. func (s *Store) ListDocuments(ctx context.Context) ([]Doc, error) { out := make([]Doc, 0) if s.projects == nil { return out, nil } projects, err := s.projects.ListProjects(ctx) if err != nil { return nil, fmt.Errorf("knowledge: list projects: %w", err) } for i := range projects { chapters, cerr := s.projects.ListChapters(ctx, projects[i].ID) if cerr != nil { s.log.Error("list chapters", "project", projects[i].ID, "err", cerr) continue } out = append(out, Doc{ ID: projects[i].ID, Name: projects[i].Title, Desc: projects[i].Desc, Chunks: len(chapters), Embedded: s.vectorOn, }) } return out, nil } // Search runs the knowledge-base search over the project index (line-level, // hierarchy-aware) and, when a semantic retriever is configured, folds in // project-linked memories ranked by the real vector path. A blank query returns // no hits. func (s *Store) Search(ctx context.Context, query string) ([]Hit, error) { hits := make([]Hit, 0) if strings.TrimSpace(query) == "" { return hits, nil } if s.projects != nil { textHits, err := s.searchProjects(ctx, query) if err != nil { return nil, err } hits = append(hits, textHits...) } if s.retriever != nil && s.defaultPersona != "" { semHits, err := s.searchSemantic(ctx, query) if err != nil { s.log.Error("semantic search", "err", err) } else { hits = append(hits, semHits...) } } return hits, nil } // searchProjects walks every project and returns chapter/item hits via // projectindex.Search (keyword + line context). A project deleted between the // list and the search is skipped rather than failing the whole search. func (s *Store) searchProjects(ctx context.Context, query string) ([]Hit, error) { projects, err := s.projects.ListProjects(ctx) if err != nil { return nil, fmt.Errorf("knowledge: list projects: %w", err) } hits := make([]Hit, 0) for i := range projects { res, err := s.projects.Search(ctx, projects[i].ID, query) if errors.Is(err, projectindex.ErrProjectNotFound) { continue } if err != nil { return nil, fmt.Errorf("knowledge: search project %q: %w", projects[i].ID, err) } if res.Project != nil { hits = append(hits, Hit{ DocID: res.Project.ID, DocName: res.Project.Title, Text: describeProject(res.Project), Score: float64(len(res.Chapters) + 1), }) } for _, ch := range res.Chapters { hits = append(hits, chapterHit(projects[i], ch)) } } return hits, nil } // searchSemantic ranks memories via the retriever's real retrieval path and maps // each project-linked result to a Hit (marked Semantic so the panel can show it). func (s *Store) searchSemantic(ctx context.Context, query string) ([]Hit, error) { results, err := s.retriever.Retrieve(ctx, s.defaultPersona, query, defaultTopK) if err != nil { return nil, fmt.Errorf("knowledge: retrieve: %w", err) } hits := make([]Hit, 0, len(results)) for _, r := range results { if r.Project == nil { continue } hits = append(hits, Hit{ DocID: r.Project.ID, DocName: r.Project.Title, Text: r.Item.Content, Score: r.Score, Semantic: true, }) } return hits, nil } // chapterHit maps a matched chapter to a Hit. Text is the located line plus its // surrounding context (projectindex.LineMatch.Context), and Score reflects how // many lines in the chapter matched. func chapterHit(p projectindex.Project, ch projectindex.ChapterHit) Hit { text := "" if len(ch.Matches) > 0 { text = ch.Matches[0].Context } return Hit{ DocID: p.ID, DocName: p.Title, Chapter: ch.Chapter.Title, Text: text, Score: float64(len(ch.Matches)), } } // describeProject summarizes a project-index hit (title/desc match) for the panel. func describeProject(p *projectindex.Project) string { if p.Desc != "" { return p.Desc } return p.Title } // IndexDocument persists a document into the project index: it creates a project // entry (title + description), chunks the content into chapters and adds each // chapter, so the document becomes immediately text/keyword-searchable via // projectindex.Search. A blank name or content returns an error. When the // project-index store is not wired there is nowhere to persist, so it returns an // error rather than a fabricated success. // // Semantic embedding is a graceful no-op: this data source does not own a vector // index, so a document imported here is keyword-searchable, and the existing // semantic path (project-linked memories ranked by the retriever) is untouched. // Per the red line that a configured vector enhancement must be genuinely used // and never a shell, we do not pretend to embed; the "Embedded" flag still // reflects the deployment-level vector capability reported by ListDocuments. func (s *Store) IndexDocument(ctx context.Context, doc Doc) error { name := strings.TrimSpace(doc.Name) if name == "" { return errors.New("knowledge: index document: name is required") } content := strings.TrimSpace(doc.Content) if content == "" { return errors.New("knowledge: index document: content is required") } if s.projects == nil { return errors.New("knowledge: index document: project index store is not configured") } projectID := strings.TrimSpace(doc.ID) if projectID == "" { id, err := newProjectID() if err != nil { return fmt.Errorf("knowledge: index document: %w", err) } projectID = id } if err := s.projects.PutProject(ctx, projectindex.Project{ ID: projectID, Title: name, Desc: strings.TrimSpace(doc.Desc), }); err != nil { return fmt.Errorf("knowledge: index project: %w", err) } chunks := chunkDocument(content) for i, ch := range chunks { if err := s.projects.AddChapter(ctx, projectID, projectindex.Chapter{ ProjectID: projectID, Title: fmt.Sprintf("第 %d 节", i+1), Content: ch, }); err != nil { return fmt.Errorf("knowledge: index chapter %d: %w", i+1, err) } } s.log.Info("indexed document", "project", projectID, "name", name, "chapters", len(chunks)) return nil } // DeleteDocument is not implemented in this milestone. It returns // ErrWriteUnsupported so the handler answers honestly (delete = M10). func (s *Store) DeleteDocument(_ context.Context, _ string) error { return ErrWriteUnsupported } // maxChapterRunes bounds a single chapter's content length. Paragraphs longer // than this are split into fixed-run pieces so no chapter is overwhelmingly large. const maxChapterRunes = 500 // chunkDocument splits content into chapter-sized strings. Paragraphs are // split on blank lines (a run of two or more newlines); a paragraph longer than // maxChapterRunes is cut into fixed-run pieces at a rune boundary. Empty // paragraphs are dropped and a non-nil (possibly empty) slice is returned. func chunkDocument(content string) []string { paragraphs := strings.Split(content, "\n\n") chunks := make([]string, 0, len(paragraphs)) for _, p := range paragraphs { p = strings.TrimSpace(p) if p == "" { continue } if utf8.RuneCountInString(p) <= maxChapterRunes { chunks = append(chunks, p) continue } chunks = append(chunks, splitRunes(p, maxChapterRunes)...) } return chunks } // splitRunes breaks s into pieces of at most n runes each. func splitRunes(s string, n int) []string { rs := []rune(s) out := make([]string, 0, (len(rs)+n-1)/n) for len(rs) > 0 { if len(rs) <= n { out = append(out, string(rs)) break } out = append(out, string(rs[:n])) rs = rs[n:] } return out } // newProjectID returns a 128-bit cryptographically random id, hex-encoded. It // avoids a UUID dependency so the package stays self-contained. func newProjectID() (string, error) { b := make([]byte, 16) if _, err := rand.Read(b); err != nil { return "", fmt.Errorf("knowledge: generate id: %w", err) } return hex.EncodeToString(b), nil }