Cyrene_v2/internal/runtime/knowledge/store.go

// Package knowledge implements the RAG knowledge-base data source described in
// DevDocs/design/10-后台管理面板.md §3.12 (module 8 知识库 RAG) and the
// knowledge/memory separation in DevDocs/design/03-记忆.md §3 (③ 知识库 =
// 参考资料, 记忆 = 经历).
//
// It reads the project index (项目索引 → 项目描述 → 章节内容) as the set of knowledge
// documents and answers searches over it with line-level context. When a semantic
// retriever is wired, it also folds in project-linked long-term memories as
// semantic hits, ranked by the retriever's real keyword × semantic × decay path —
// so a configured vector index (pgvector / SQLite vector) is genuinely used and
// never a shell.
//
// The store persists knowledge documents by indexing them into the project index
// (项目索引 → 项目描述 → 章节内容): IndexDocument chunks a document's content and
// stores it as a project plus its chapters, making it text/keyword-searchable
// immediately. Semantic/vector embedding of an imported document is a graceful
// no-op here (this data source holds no vector index); the existing semantic path
// still ranks project-linked memories via the retriever. DeleteDocument removes a
// document (its project entry and, in one transaction, every chapter that
// references it) so the import/delete pair is a real round-trip.
package knowledge

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"strings"
	"unicode/utf8"

	"git.yeij.top/AskaEth/Cyrene_v2/internal/logger"
	"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory"
	"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/memory/projectindex"
)

// ErrWriteUnsupported is returned by a knowledge store implementation that does
// not support document deletion. *Store implements DeleteDocument, so it never
// returns this; it remains for alternate/degraded implementations the admin
// handler maps to a clear 501 rather than a fabricated success.
var ErrWriteUnsupported = errors.New("knowledge: delete is not supported by this store")

// ErrDocumentNotFound is returned by DeleteDocument when the id is blank or no
// knowledge document (project-index entry) carries it.
var ErrDocumentNotFound = errors.New("knowledge: document not found")

// defaultTopK bounds the semantic retrieval path.
const defaultTopK = 8

// Retriever is the subset of the memory retriever the knowledge store uses for
// semantic ranking. *memory.Retriever implements it; nil means no semantic path.
type Retriever interface {
	Retrieve(ctx context.Context, personaID, query string, topK int) ([]memory.Result, error)
}

// Doc is one knowledge-base document backing a project-index entry.
type Doc struct {
	// ID is the project index id.
	ID string
	// Name is the project title (the document's display name).
	Name string
	// Desc is the project description (layer 2 of the hierarchy).
	Desc string
	// Chunks is the number of chapters the document is split into.
	Chunks int
	// Embedded reports whether a persistent vector index is wired for this
	// deployment. This is the deployment-level semantic capability; per-document
	// embedding state is not tracked yet.
	Embedded bool
	// Content is the document body supplied to IndexDocument. It is empty for
	// documents returned by ListDocuments (they carry metadata only) and is
	// chunked into chapters by IndexDocument.
	Content string
}

// Hit is one retrieval result surfaced to the admin panel.
type Hit struct {
	// DocID is the owning project index id.
	DocID string
	// DocName is the owning project title.
	DocName string
	// Chapter is the chapter title that matched (empty for a semantic/memory hit).
	Chapter string
	// Text is the matched content: a chapter's located line(s) with context, or
	// the memory content for a semantic hit.
	Text string
	// Score is the relevance score (higher = more relevant).
	Score float64
	// Semantic reports whether the hit was ranked by the semantic/vector path.
	Semantic bool
}

// Deps carries the construction inputs. Every field is nil-tolerant so a store
// without a project index or retriever degrades to read-only empty answers.
type Deps struct {
	// Projects is the project index backing the document list and text search.
	// Optional; when nil the document list and text search return empty.
	Projects *projectindex.Store
	// Retriever ranks project-linked memories semantically. Optional; the
	// semantic path runs only when both Retriever and DefaultPersona are set.
	Retriever Retriever
	// DefaultPersona scopes semantic retrieval to one persona. Optional.
	DefaultPersona string
	// VectorOn reports whether a persistent vector index (pgvector / SQLite
	// vector) is wired, so Doc.Embedded is honest and never a shell.
	VectorOn bool
	// Log is the base logger. Nil uses a discard logger tagged with this package.
	Log *slog.Logger
}

// Store is the RAG data source. It is stateless and safe for concurrent use.
type Store struct {
	projects       *projectindex.Store
	retriever      Retriever
	defaultPersona string
	vectorOn       bool
	log            *slog.Logger
}

// New builds a Store. Availability degrades gracefully: a nil project index means
// no documents, a nil retriever means keyword/line search only.
func New(d Deps) *Store {
	log := d.Log
	if log == nil {
		log = slog.New(slog.NewTextHandler(io.Discard, nil))
	}
	return &Store{
		projects:       d.Projects,
		retriever:      d.Retriever,
		defaultPersona: strings.TrimSpace(d.DefaultPersona),
		vectorOn:       d.VectorOn,
		log:            logger.WithSource(log, "runtime/knowledge"),
	}
}

// ListDocuments enumerates the knowledge-base documents (project-index entries).
// A nil project index returns an empty, non-nil slice.
func (s *Store) ListDocuments(ctx context.Context) ([]Doc, error) {
	out := make([]Doc, 0)
	if s.projects == nil {
		return out, nil
	}
	projects, err := s.projects.ListProjects(ctx)
	if err != nil {
		return nil, fmt.Errorf("knowledge: list projects: %w", err)
	}
	for i := range projects {
		chapters, cerr := s.projects.ListChapters(ctx, projects[i].ID)
		if cerr != nil {
			s.log.Error("list chapters", "project", projects[i].ID, "err", cerr)
			continue
		}
		out = append(out, Doc{
			ID:       projects[i].ID,
			Name:     projects[i].Title,
			Desc:     projects[i].Desc,
			Chunks:   len(chapters),
			Embedded: s.vectorOn,
		})
	}
	return out, nil
}

// Search runs the knowledge-base search over the project index (line-level,
// hierarchy-aware) and, when a semantic retriever is configured, folds in
// project-linked memories ranked by the real vector path. A blank query returns
// no hits.
func (s *Store) Search(ctx context.Context, query string) ([]Hit, error) {
	hits := make([]Hit, 0)
	if strings.TrimSpace(query) == "" {
		return hits, nil
	}
	if s.projects != nil {
		textHits, err := s.searchProjects(ctx, query)
		if err != nil {
			return nil, err
		}
		hits = append(hits, textHits...)
	}
	if s.retriever != nil && s.defaultPersona != "" {
		semHits, err := s.searchSemantic(ctx, query)
		if err != nil {
			s.log.Error("semantic search", "err", err)
		} else {
			hits = append(hits, semHits...)
		}
	}
	return hits, nil
}

// searchProjects walks every project and returns chapter/item hits via
// projectindex.Search (keyword + line context). A project deleted between the
// list and the search is skipped rather than failing the whole search.
func (s *Store) searchProjects(ctx context.Context, query string) ([]Hit, error) {
	projects, err := s.projects.ListProjects(ctx)
	if err != nil {
		return nil, fmt.Errorf("knowledge: list projects: %w", err)
	}
	hits := make([]Hit, 0)
	for i := range projects {
		res, err := s.projects.Search(ctx, projects[i].ID, query)
		if errors.Is(err, projectindex.ErrProjectNotFound) {
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("knowledge: search project %q: %w", projects[i].ID, err)
		}
		if res.Project != nil {
			hits = append(hits, Hit{
				DocID:   res.Project.ID,
				DocName: res.Project.Title,
				Text:    describeProject(res.Project),
				Score:   float64(len(res.Chapters) + 1),
			})
		}
		for _, ch := range res.Chapters {
			hits = append(hits, chapterHit(projects[i], ch))
		}
	}
	return hits, nil
}

// searchSemantic ranks memories via the retriever's real retrieval path and maps
// each project-linked result to a Hit (marked Semantic so the panel can show it).
func (s *Store) searchSemantic(ctx context.Context, query string) ([]Hit, error) {
	results, err := s.retriever.Retrieve(ctx, s.defaultPersona, query, defaultTopK)
	if err != nil {
		return nil, fmt.Errorf("knowledge: retrieve: %w", err)
	}
	hits := make([]Hit, 0, len(results))
	for _, r := range results {
		if r.Project == nil {
			continue
		}
		hits = append(hits, Hit{
			DocID:    r.Project.ID,
			DocName:  r.Project.Title,
			Text:     r.Item.Content,
			Score:    r.Score,
			Semantic: true,
		})
	}
	return hits, nil
}

// chapterHit maps a matched chapter to a Hit. Text is the located line plus its
// surrounding context (projectindex.LineMatch.Context), and Score reflects how
// many lines in the chapter matched.
func chapterHit(p projectindex.Project, ch projectindex.ChapterHit) Hit {
	text := ""
	if len(ch.Matches) > 0 {
		text = ch.Matches[0].Context
	}
	return Hit{
		DocID:   p.ID,
		DocName: p.Title,
		Chapter: ch.Chapter.Title,
		Text:    text,
		Score:   float64(len(ch.Matches)),
	}
}

// describeProject summarizes a project-index hit (title/desc match) for the panel.
func describeProject(p *projectindex.Project) string {
	if p.Desc != "" {
		return p.Desc
	}
	return p.Title
}

// IndexDocument persists a document into the project index: it creates a project
// entry (title + description), chunks the content into chapters and adds each
// chapter, so the document becomes immediately text/keyword-searchable via
// projectindex.Search. A blank name or content returns an error. When the
// project-index store is not wired there is nowhere to persist, so it returns an
// error rather than a fabricated success.
//
// Semantic embedding is a graceful no-op: this data source does not own a vector
// index, so a document imported here is keyword-searchable, and the existing
// semantic path (project-linked memories ranked by the retriever) is untouched.
// Per the red line that a configured vector enhancement must be genuinely used
// and never a shell, we do not pretend to embed; the "Embedded" flag still
// reflects the deployment-level vector capability reported by ListDocuments.
func (s *Store) IndexDocument(ctx context.Context, doc Doc) error {
	name := strings.TrimSpace(doc.Name)
	if name == "" {
		return errors.New("knowledge: index document: name is required")
	}
	content := strings.TrimSpace(doc.Content)
	if content == "" {
		return errors.New("knowledge: index document: content is required")
	}
	if s.projects == nil {
		return errors.New("knowledge: index document: project index store is not configured")
	}

	projectID := strings.TrimSpace(doc.ID)
	if projectID == "" {
		id, err := newProjectID()
		if err != nil {
			return fmt.Errorf("knowledge: index document: %w", err)
		}
		projectID = id
	}

	if err := s.projects.PutProject(ctx, projectindex.Project{
		ID:    projectID,
		Title: name,
		Desc:  strings.TrimSpace(doc.Desc),
	}); err != nil {
		return fmt.Errorf("knowledge: index project: %w", err)
	}

	chunks := chunkDocument(content)
	for i, ch := range chunks {
		if err := s.projects.AddChapter(ctx, projectID, projectindex.Chapter{
			ProjectID: projectID,
			Title:     fmt.Sprintf("第 %d 节", i+1),
			Content:   ch,
		}); err != nil {
			return fmt.Errorf("knowledge: index chapter %d: %w", i+1, err)
		}
	}
	s.log.Info("indexed document", "project", projectID, "name", name, "chapters", len(chunks))
	return nil
}

// DeleteDocument removes a knowledge document: the project-index entry and, in
// the same transaction, every chapter that references it (projectindex.DeleteProject
// cascades the chunk cleanup). It returns ErrDocumentNotFound for a blank id or
// one with no project entry, and an error when no project index is wired (there
// is nowhere to delete from, so a fabricated success is never returned).
func (s *Store) DeleteDocument(ctx context.Context, id string) error {
	docID := strings.TrimSpace(id)
	if docID == "" {
		return fmt.Errorf("knowledge: delete document: %w", ErrDocumentNotFound)
	}
	if s.projects == nil {
		return errors.New("knowledge: delete document: project index store is not configured")
	}
	if err := s.projects.DeleteProject(ctx, docID); err != nil {
		if errors.Is(err, projectindex.ErrProjectNotFound) {
			return fmt.Errorf("knowledge: delete document %q: %w", docID, ErrDocumentNotFound)
		}
		return fmt.Errorf("knowledge: delete document %q: %w", docID, err)
	}
	s.log.Info("deleted document", "project", docID)
	return nil
}

// maxChapterRunes bounds a single chapter's content length. Paragraphs longer
// than this are split into fixed-run pieces so no chapter is overwhelmingly large.
const maxChapterRunes = 500

// chunkDocument splits content into chapter-sized strings. Paragraphs are
// split on blank lines (a run of two or more newlines); a paragraph longer than
// maxChapterRunes is cut into fixed-run pieces at a rune boundary. Empty
// paragraphs are dropped and a non-nil (possibly empty) slice is returned.
func chunkDocument(content string) []string {
	paragraphs := strings.Split(content, "\n\n")
	chunks := make([]string, 0, len(paragraphs))
	for _, p := range paragraphs {
		p = strings.TrimSpace(p)
		if p == "" {
			continue
		}
		if utf8.RuneCountInString(p) <= maxChapterRunes {
			chunks = append(chunks, p)
			continue
		}
		chunks = append(chunks, splitRunes(p, maxChapterRunes)...)
	}
	return chunks
}

// splitRunes breaks s into pieces of at most n runes each.
func splitRunes(s string, n int) []string {
	rs := []rune(s)
	out := make([]string, 0, (len(rs)+n-1)/n)
	for len(rs) > 0 {
		if len(rs) <= n {
			out = append(out, string(rs))
			break
		}
		out = append(out, string(rs[:n]))
		rs = rs[n:]
	}
	return out
}

// newProjectID returns a 128-bit cryptographically random id, hex-encoded. It
// avoids a UUID dependency so the package stays self-contained.
func newProjectID() (string, error) {
	b := make([]byte, 16)
	if _, err := rand.Read(b); err != nil {
		return "", fmt.Errorf("knowledge: generate id: %w", err)
	}
	return hex.EncodeToString(b), nil
}