docs(comments): 注释中文化 · internal/runtime/memory(31 文件,纯替换)
- 只翻注释/文档字符串:820 增 / 820 删 严格 1:1;剥掉整行注释与行尾注释尾巴后, 逐文件与 HEAD 比对「代码部分差异 = 0」;裁决/design 编号计数逐项与 HEAD 一致 - 保留可执行 SQL 样例、SQLite 报错原文、pgvector/HNSW/ANN/BLOB 等技术词原文 - 复核:go build 绿 + memory/projectindex/vector 三包测试绿;gofmt 仅剩 2 个既有未格式化文件(改动前即如此,未顺手修)
This commit is contained in:
1 parent
eb1828ee7a
commit
61cf80d5ee
31 files changed
+820
-820
No files matched your search
@@ -1,22 +1,22 @@
|
||||
// Package vector implements dense-vector similarity search for Cyrene's
|
||||
// long-term memory layer (DevDocs/design/03-记忆.md §3).
|
||||
// vector 包为 Cyrene 的长期记忆层(DevDocs/design/03-记忆.md §3)
|
||||
// 实现稠密向量相似度检索。
|
||||
//
|
||||
// It exposes an Index abstraction over two storage backends:
|
||||
// 它在两种存储后端之上暴露 Index 抽象:
|
||||
//
|
||||
// - SQLiteIndex (the default): vectors are stored as BLOBs in a SQLite
|
||||
// vec_items table and similarity is computed in Go as cosine similarity via
|
||||
// a linear scan over the persisted vectors. This is the zero-extra-
|
||||
// dependency path used when DB_PG_URL is empty.
|
||||
// - PGVectorIndex (optional enhancement): uses PostgreSQL + pgvector, storing
|
||||
// vectors in a vector column and answering similarity with the cosine-
|
||||
// distance operator <=> in SQL. It is selected only when DB_PG_URL is set;
|
||||
// when it is, semantic search is genuinely backed by pgvector and does not
|
||||
// silently fall back to keyword matching.
|
||||
// - SQLiteIndex(默认):向量以 BLOB 存在 SQLite 的
|
||||
// vec_items 表中,相似度在 Go 侧通过对已持久化向量
|
||||
// 线性扫描计算余弦相似度。这是 DB_PG_URL 为空时使用的
|
||||
// 零额外依赖路径。
|
||||
// - PGVectorIndex(可选增强):使用 PostgreSQL + pgvector,把
|
||||
// 向量存入 vector 列,并用 SQL 中的余弦
|
||||
// 距离运算符 <=> 作答相似度。仅当设置了 DB_PG_URL 时才选中;
|
||||
// 此时语义检索真正由 pgvector 支撑,不会
|
||||
// 静默退化为关键词匹配。
|
||||
//
|
||||
// Both backends are linear-scan for milestone M4 (small datasets, no ANN
|
||||
// training). The PostgreSQL path may later be upgraded to an HNSW index as data
|
||||
// grows; the Search SQL is unchanged. Vectors are produced by an
|
||||
// embedding.Embedder wired in via New.
|
||||
// 两个后端在 M4 里程碑都是线性扫描(数据集小,不做 ANN
|
||||
// 训练)。随着数据增长,PostgreSQL 路径日后可升级为 HNSW 索引;
|
||||
// Search SQL 不变。向量由经 New 接入的
|
||||
// embedding.Embedder 产出。
|
||||
package vector
|
||||
|
||||
import (
|
||||
@@ -33,22 +33,22 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
|
||||
)
|
||||
|
||||
// Vector is a dense embedding: one float32 per dimension.
|
||||
// Vector 是一个稠密向量:每维一个 float32。
|
||||
type Vector []float32
|
||||
|
||||
// Hit is a single similarity result: a stored id and its similarity score.
|
||||
// Higher Score means more similar, so results sort in descending Score order.
|
||||
// Hit 是一条相似度结果:一个已存 id 及其相似度得分。
|
||||
// Score 越高越相似,因此结果按 Score 降序排列。
|
||||
type Hit struct {
|
||||
ID string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// Index is a dense-vector similarity index over stored (id, vector) pairs.
|
||||
// Index 是建立在已存 (id, vector) 对上的稠密向量相似度索引。
|
||||
//
|
||||
// Embed turns a batch of texts into vectors using the wired embedder. Add
|
||||
// persists an (id, vector) pair, upserting on a repeated id. Search returns the
|
||||
// topK ids most similar to q, ranked by descending similarity. Dim reports the
|
||||
// fixed vector dimension. Close releases the underlying database.
|
||||
// Embed 用接入的 embedder 把一批文本转成向量。Add
|
||||
// 持久化一个 (id, vector) 对,id 重复时 upsert。Search 返回与 q
|
||||
// 最相似的前 topK 个 id,按相似度降序排列。Dim 报告
|
||||
// 固定向量维度。Close 释放底层数据库。
|
||||
type Index interface {
|
||||
Embed(ctx context.Context, texts []string) ([]Vector, error)
|
||||
Add(ctx context.Context, id string, v Vector) error
|
||||
@@ -57,51 +57,51 @@ type Index interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Package-level sentinel errors. Each is wrapped with %w at the call site so
|
||||
// callers can use errors.Is.
|
||||
// 包级哨兵错误。每个都在调用点用 %w 包裹,调用方
|
||||
// 可用 errors.Is 判断。
|
||||
var (
|
||||
// ErrNoEmbedder is returned when an index is asked to Embed but no embedder
|
||||
// has been wired in (e.g. a constructor called directly rather than via New).
|
||||
// ErrNoEmbedder 在索引被要求 Embed 却没有接入 embedder
|
||||
// 时返回(例如直接调用构造函数而非经 New)。
|
||||
ErrNoEmbedder = errors.New("memory/vector: no embedder configured")
|
||||
|
||||
// ErrNilDB is returned when a constructor receives a nil *sql.DB.
|
||||
// ErrNilDB 在构造函数收到 nil *sql.DB 时返回。
|
||||
ErrNilDB = errors.New("memory/vector: nil db")
|
||||
|
||||
// ErrInvalidDim is returned when a dimension is not positive.
|
||||
// ErrInvalidDim 在维度非正时返回。
|
||||
ErrInvalidDim = errors.New("memory/vector: dimension must be positive")
|
||||
|
||||
// ErrDimMismatch is returned when an added or queried vector's length does
|
||||
// not match the index dimension.
|
||||
// ErrDimMismatch 在新增或查询的向量长度与
|
||||
// 索引维度不匹配时返回。
|
||||
ErrDimMismatch = errors.New("memory/vector: vector dimension mismatch")
|
||||
|
||||
// ErrMissingID is returned when an id is blank.
|
||||
// ErrMissingID 在 id 为空时返回。
|
||||
ErrMissingID = errors.New("memory/vector: id is required")
|
||||
|
||||
// ErrInvalidPGURL is returned when the PostgreSQL URL is blank.
|
||||
// ErrInvalidPGURL 在 PostgreSQL URL 为空时返回。
|
||||
ErrInvalidPGURL = errors.New("memory/vector: invalid postgres url")
|
||||
)
|
||||
|
||||
// New builds the Index selected by cfg: PGVectorIndex when cfg.PGURL is set,
|
||||
// otherwise SQLiteIndex. embedder must be non-nil and report a positive Dim();
|
||||
// it is wired into the returned index so Index.Embed produces vectors with it.
|
||||
// log may be nil; logs are tagged with source "memory/vector".
|
||||
// New 构建 cfg 选定的 Index:设置了 cfg.PGURL 时为 PGVectorIndex,
|
||||
// 否则为 SQLiteIndex。embedder 必须非 nil 且 Dim() 为正;
|
||||
// 它被接入返回的索引,使 Index.Embed 用它产向量。
|
||||
// log 可为 nil;日志带 source "memory/vector" 标签。
|
||||
//
|
||||
// SQLiteIndex is created on a connection opened at cfg.SQLitePath (":memory:"
|
||||
// for an isolated test database or a file path). PGVectorIndex connects to
|
||||
// cfg.PGURL with the pgx driver and requires a PostgreSQL server with the
|
||||
// pgvector extension installed.
|
||||
// SQLiteIndex 建立在 cfg.SQLitePath 打开的连接上(":memory:"
|
||||
// 用于隔离的测试数据库,或一个文件路径)。PGVectorIndex 用
|
||||
// pgx 驱动连接 cfg.PGURL,要求已安装
|
||||
// pgvector 扩展的 PostgreSQL 服务。
|
||||
//
|
||||
// New performs one-time setup with a background context and has no ctx
|
||||
// parameter by design; every subsequent operation accepts a context.Cancel.
|
||||
// New 用后台 context 做一次性准备,按设计不接收 ctx
|
||||
// 参数;后续每个操作都接受 context.Cancel。
|
||||
func New(cfg config.DBConfig, embedder embedding.Embedder, log *slog.Logger) (Index, error) {
|
||||
if embedder == nil {
|
||||
return nil, ErrNoEmbedder
|
||||
}
|
||||
dim := embedder.Dim()
|
||||
if dim <= 0 {
|
||||
// API-backed embedders do not know their vector dimension until the first
|
||||
// successful response. Probe once so the index is built with the real
|
||||
// dimension instead of erroring out and falling back to in-memory.
|
||||
// 走 API 的 embedder 在首次成功响应前不知道自己的向量维度。
|
||||
// 探测一次,使索引以真实
|
||||
// 维度构建,而不是报错并回退到内存内实现。
|
||||
if _, err := embedder.Embed(context.Background(), []string{"probe"}); err != nil {
|
||||
return nil, fmt.Errorf("memory/vector: probe embedder dim: %w", err)
|
||||
}
|
||||
@@ -118,8 +118,8 @@ func New(cfg config.DBConfig, embedder embedding.Embedder, log *slog.Logger) (In
|
||||
idx.withEmbedder(embedder)
|
||||
return idx, nil
|
||||
}
|
||||
// db.Open logs internally; hand it a sourced logger so its records carry
|
||||
// provenance, while the raw log goes to NewSQLite which sources it itself.
|
||||
// db.Open 内部会记日志;交给它一个带来源的 logger,使其记录携带
|
||||
// 出处,而原始 log 传给 NewSQLite,由后者自行加来源。
|
||||
d, err := db.Open(context.Background(), cfg.SQLitePath, sourcedLogger(log))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("memory/vector: open sqlite: %w", err)
|
||||
@@ -133,16 +133,16 @@ func New(cfg config.DBConfig, embedder embedding.Embedder, log *slog.Logger) (In
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// usePG reports whether cfg should select the PostgreSQL+pgvector backend.
|
||||
// Surrounding whitespace counts as unset.
|
||||
// usePG 报告 cfg 是否应选择 PostgreSQL+pgvector 后端。
|
||||
// 首尾空白视为未设置。
|
||||
func usePG(pgURL string) bool {
|
||||
return strings.TrimSpace(pgURL) != ""
|
||||
}
|
||||
|
||||
// sourcedLogger returns log, or a discard logger when nil, tagged with source
|
||||
// "memory/vector" so every record carries provenance. It is idempotent in
|
||||
// output: re-applying the same source value on an already-sourced logger yields
|
||||
// the same emitted attribute.
|
||||
// sourcedLogger 返回 log,nil 时返回丢弃型 logger,并打上 source
|
||||
// "memory/vector" 标签,使每条记录都携带出处。它在
|
||||
// 输出上幂等:对已带来源的 logger 再次施加相同 source 值,
|
||||
// 得到的输出属性相同。
|
||||
func sourcedLogger(log *slog.Logger) *slog.Logger {
|
||||
if log == nil {
|
||||
log = slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
|
||||
@@ -13,14 +13,14 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
|
||||
)
|
||||
|
||||
// testLogger returns a logger that writes nowhere, keeping test output clean.
|
||||
// testLogger 返回一个不写任何地方的 logger,保持测试输出干净。
|
||||
func testLogger() *slog.Logger {
|
||||
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
}
|
||||
|
||||
// openTestDB opens an isolated in-memory SQLite DB. Each call owns a private DB,
|
||||
// so parallel tests are isolated by construction. The returned DB is closed by
|
||||
// t.Cleanup.
|
||||
// openTestDB 打开一个隔离的内存 SQLite 库。每次调用都拥有私有 DB,
|
||||
// 因此并行测试天生相互隔离。返回的 DB 由
|
||||
// t.Cleanup 关闭。
|
||||
func openTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
@@ -32,8 +32,8 @@ func openTestDB(t *testing.T) *sql.DB {
|
||||
return d
|
||||
}
|
||||
|
||||
// newInMemorySQLite builds a SQLiteIndex on a fresh in-memory DB with dim
|
||||
// dimensions. The embedder is left unwired so tests control it.
|
||||
// newInMemorySQLite 在全新的内存 DB 上构建维度为 dim 的 SQLiteIndex。
|
||||
// embedder 不接入,由测试自行控制。
|
||||
func newInMemorySQLite(t *testing.T, dim int) *SQLiteIndex {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
@@ -44,8 +44,8 @@ func newInMemorySQLite(t *testing.T, dim int) *SQLiteIndex {
|
||||
return idx
|
||||
}
|
||||
|
||||
// mockEmbedder is a scriptable embedding.Embedder for tests. With no fn it
|
||||
// returns one-hot vectors, one per input text, so tests can predict results.
|
||||
// mockEmbedder 是测试用的可脚本化 embedding.Embedder。未设置 fn 时它
|
||||
// 返回 one-hot 向量,每段输入文本一个,测试因此可预测结果。
|
||||
type mockEmbedder struct {
|
||||
dim int
|
||||
fn func(ctx context.Context, texts []string) ([]embedding.Vector, error)
|
||||
@@ -98,7 +98,7 @@ func TestNew_SelectsSQLite_WhenPGURLBlank(t *testing.T) {
|
||||
if _, ok := idx.(*SQLiteIndex); !ok {
|
||||
t.Fatalf("New type = %T, want *SQLiteIndex", idx)
|
||||
}
|
||||
// New wires the embedder, so Embed works on the returned index.
|
||||
// New 接入了 embedder,因此返回的索引上 Embed 可用。
|
||||
vs, err := idx.Embed(ctx, []string{"hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("Embed: %v", err)
|
||||
@@ -127,9 +127,9 @@ func TestNew_PGURLWhitespace_SelectsSQLite(t *testing.T) {
|
||||
|
||||
func TestNew_PGSelection_AttemptsPGWhenURLSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
// A loopback URL on a closed port fails fast, proving New chose the PG path
|
||||
// rather than silently falling back to SQLite. Port 1 (tcpmux) does not
|
||||
// listen on localhost, so the connection is refused immediately.
|
||||
// 指向已关闭端口的环回 URL 会快速失败,证明 New 选择了 PG 路径
|
||||
// 而不是静默回退到 SQLite。端口 1(tcpmux)在
|
||||
// localhost 上不监听,因此连接立即被拒。
|
||||
idx, err := New(config.DBConfig{
|
||||
SQLitePath: ":memory:",
|
||||
PGURL: "postgres://127.0.0.1:1/mem?sslmode=disable&connect_timeout=1",
|
||||
|
||||
@@ -10,25 +10,25 @@ import (
|
||||
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // Postgres driver, registers name "pgx"
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // Postgres 驱动,注册名为 "pgx"
|
||||
)
|
||||
|
||||
// pgMaxOpenConns is the PostgreSQL connection pool size mandated by
|
||||
// DevDocs/02-development-constraints.md §2.2 for the pgvector enhancement.
|
||||
// pgMaxOpenConns 是 DevDocs/02-development-constraints.md §2.2 为
|
||||
// pgvector 增强规定的 PostgreSQL 连接池大小。
|
||||
const pgMaxOpenConns = 25
|
||||
|
||||
// PGVectorIndex stores vectors in a PostgreSQL pgvector vector column and
|
||||
// answers similarity with the cosine-distance operator <=> in SQL. It is the
|
||||
// optional enhancement selected when DB_PG_URL is configured; when used,
|
||||
// semantic search is genuinely backed by pgvector and does not fall back to
|
||||
// keyword matching (DevDocs/design/03-记忆.md §3).
|
||||
// PGVectorIndex 把向量存入 PostgreSQL 的 pgvector vector 列,并
|
||||
// 用 SQL 中的余弦距离运算符 <=> 作答相似度。它是配置了
|
||||
// DB_PG_URL 时选中的可选增强;启用时,
|
||||
// 语义检索真正由 pgvector 支撑,不会退化为
|
||||
// 关键词匹配(DevDocs/design/03-记忆.md §3)。
|
||||
//
|
||||
// The schema is prepared by NewPG (vector extension + vec_items table). Search
|
||||
// issues ORDER BY vector <=> $1 LIMIT n, which returns exact cosine-distance-
|
||||
// ranked results. With no ANN index the query planner does a sequential scan,
|
||||
// which is the linear-scan behavior expected for milestone M4 (small datasets).
|
||||
// An HNSW index (CREATE INDEX ... USING hnsw (vector vector_cosine_ops)) is the
|
||||
// documented ANN upgrade for larger datasets and requires no change to Search.
|
||||
// schema 由 NewPG 准备(vector 扩展 + vec_items 表)。Search
|
||||
// 发出 ORDER BY vector <=> $1 LIMIT n,返回精确的余弦距离
|
||||
// 排名结果。没有 ANN 索引时,查询规划器做顺序扫描,
|
||||
// 这正是 M4 里程碑(小数据集)期望的线性扫描行为。
|
||||
// HNSW 索引(CREATE INDEX ... USING hnsw (vector vector_cosine_ops))是
|
||||
// 大数据集下既定的 ANN 升级方案,且无需改动 Search。
|
||||
type PGVectorIndex struct {
|
||||
db *sql.DB
|
||||
dim int
|
||||
@@ -36,16 +36,16 @@ type PGVectorIndex struct {
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewPG connects to the PostgreSQL+pgvector database at pgURL using the pgx
|
||||
// driver and prepares the schema: the vector extension and a vec_items table
|
||||
// with a fixed-dimension vector column. It deliberately creates no ANN index:
|
||||
// for milestone M4 the <=> query runs as an exact sequential scan, and an HNSW
|
||||
// index is a documented, optional upgrade for larger data. A blank pgURL
|
||||
// returns ErrInvalidPGURL, and a non-positive dim returns ErrInvalidDim, both
|
||||
// before any network I/O. ctx controls the connection ping and schema setup.
|
||||
// NewPG 用 pgx 驱动连接到 pgURL 的 PostgreSQL+pgvector 数据库
|
||||
// 并准备 schema:vector 扩展与一张带固定维度 vector 列的
|
||||
// vec_items 表。它刻意不创建 ANN 索引:
|
||||
// M4 里程碑下 <=> 查询走精确顺序扫描,而 HNSW
|
||||
// 索引是既定的大数据量可选升级。pgURL 为空
|
||||
// 返回 ErrInvalidPGURL,dim 非正返回 ErrInvalidDim,两者都在
|
||||
// 任何网络 I/O 之前。ctx 控制连接 ping 与 schema 准备。
|
||||
//
|
||||
// The returned index has no embedder wired in; use New to build a fully wired
|
||||
// index, or wire one in before calling Embed.
|
||||
// 返回的索引没有接入 embedder;用 New 构建完全接好的
|
||||
// 索引,或在调用 Embed 前接入一个。
|
||||
func NewPG(ctx context.Context, pgURL string, dim int, log *slog.Logger) (*PGVectorIndex, error) {
|
||||
if strings.TrimSpace(pgURL) == "" {
|
||||
return nil, ErrInvalidPGURL
|
||||
@@ -74,16 +74,16 @@ func NewPG(ctx context.Context, pgURL string, dim int, log *slog.Logger) (*PGVec
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// setup creates the pgvector extension and the vec_items table idempotently.
|
||||
// For milestone M4 no ANN index is created — <->/<=> queries fall back to an
|
||||
// exact sequential scan, matching the linear-scan semantics of the SQLite path.
|
||||
// To scale, add:
|
||||
// setup 幂等地创建 pgvector 扩展与 vec_items 表。
|
||||
// M4 里程碑不创建 ANN 索引 —— <->/<=> 查询退化为
|
||||
// 精确顺序扫描,与 SQLite 路径的线性扫描语义一致。
|
||||
// 要扩容,可添加:
|
||||
//
|
||||
// CREATE INDEX IF NOT EXISTS idx_vec_items_vector
|
||||
// ON vec_items USING hnsw (vector vector_cosine_ops)
|
||||
//
|
||||
// which requires no change to Search. Each statement is parameterized where it
|
||||
// takes input; the table DDL is static after dimension substitution.
|
||||
// 且无需改动 Search。每条语句在取输入处都参数化;
|
||||
// 表 DDL 在维度替换后是静态的。
|
||||
func (p *PGVectorIndex) setup(ctx context.Context) error {
|
||||
statements := []string{
|
||||
"CREATE EXTENSION IF NOT EXISTS vector",
|
||||
@@ -101,16 +101,16 @@ func (p *PGVectorIndex) setup(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// withEmbedder wires the vector-producing embedder into the index. Used by New;
|
||||
// tests may also call it directly.
|
||||
// withEmbedder 把产向量的 embedder 接入索引。New 使用它;
|
||||
// 测试也可直接调用。
|
||||
func (p *PGVectorIndex) withEmbedder(e embedding.Embedder) { p.embedder = e }
|
||||
|
||||
// Dim returns the fixed vector dimension.
|
||||
// Dim 返回固定向量维度。
|
||||
func (p *PGVectorIndex) Dim() int { return p.dim }
|
||||
|
||||
// Embed turns texts into vectors with the wired embedder, returning one Vector
|
||||
// per input text aligned by index. With no embedder wired it returns
|
||||
// ErrNoEmbedder.
|
||||
// Embed 用接入的 embedder 把 texts 转成向量,每段输入文本
|
||||
// 对应一个按索引对齐的 Vector。未接入 embedder 时返回
|
||||
// ErrNoEmbedder。
|
||||
func (p *PGVectorIndex) Embed(ctx context.Context, texts []string) ([]Vector, error) {
|
||||
if p.embedder == nil {
|
||||
return nil, ErrNoEmbedder
|
||||
@@ -126,10 +126,10 @@ func (p *PGVectorIndex) Embed(ctx context.Context, texts []string) ([]Vector, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Add persists the (id, vector) pair, upserting on a repeated id. The vector is
|
||||
// passed to Postgres as a vector literal bound to a $n::vector parameter. It
|
||||
// rejects a blank id (ErrMissingID) and a vector whose length differs from the
|
||||
// index dimension (ErrDimMismatch).
|
||||
// Add 持久化 (id, vector) 对,id 重复时 upsert。向量以
|
||||
// 绑定到 $n::vector 参数的向量字面量传给 Postgres。它
|
||||
// 拒绝空 id(ErrMissingID)以及与索引维度不一致的
|
||||
// 向量(ErrDimMismatch)。
|
||||
func (p *PGVectorIndex) Add(ctx context.Context, id string, v Vector) error {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return ErrMissingID
|
||||
@@ -147,10 +147,10 @@ func (p *PGVectorIndex) Add(ctx context.Context, id string, v Vector) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search returns up to topK ids most similar to q, ranked by descending cosine
|
||||
// similarity (1 - cosine distance), computed by pgvector in SQL. A non-positive
|
||||
// topK returns an empty result; a query whose length differs from the index
|
||||
// dimension returns ErrDimMismatch.
|
||||
// Search 返回与 q 最相似的最多 topK 个 id,按余弦相似度
|
||||
// (1 - 余弦距离)降序排列,由 pgvector 在 SQL 中计算。topK 非正
|
||||
// 返回空结果;长度与索引维度不一致的查询
|
||||
// 返回 ErrDimMismatch。
|
||||
func (p *PGVectorIndex) Search(ctx context.Context, q Vector, topK int) ([]Hit, error) {
|
||||
if topK <= 0 {
|
||||
return []Hit{}, nil
|
||||
@@ -184,7 +184,7 @@ func (p *PGVectorIndex) Search(ctx context.Context, q Vector, topK int) ([]Hit,
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying PostgreSQL connection pool.
|
||||
// Close 释放底层的 PostgreSQL 连接池。
|
||||
func (p *PGVectorIndex) Close() error {
|
||||
if p.db == nil {
|
||||
return nil
|
||||
@@ -196,9 +196,9 @@ func (p *PGVectorIndex) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pgVectorLiteral renders v in pgvector's text form "[a,b,c]" so it can be
|
||||
// bound as a vector parameter ($n::vector). Values are formatted with enough
|
||||
// precision to round-trip a float32.
|
||||
// pgVectorLiteral 把 v 渲染成 pgvector 的文本形式 "[a,b,c]",
|
||||
// 以便绑定为向量参数($n::vector)。取值的格式化精度足以
|
||||
// 往返一个 float32。
|
||||
func pgVectorLiteral(v Vector) string {
|
||||
var b strings.Builder
|
||||
b.WriteByte('[')
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestNewPG_InvalidURL_异常(t *testing.T) {
|
||||
|
||||
func TestNewPG_InvalidDim_异常(t *testing.T) {
|
||||
t.Parallel()
|
||||
// dim is validated before any connection is opened, so no network I/O here.
|
||||
// dim 在任何连接打开前就校验,因此这里没有网络 I/O。
|
||||
if _, err := NewPG(context.Background(), "postgres://host/db", 0, testLogger()); !errors.Is(err, ErrInvalidDim) {
|
||||
t.Fatalf("NewPG(dim 0) err = %v, want ErrInvalidDim", err)
|
||||
}
|
||||
@@ -44,11 +44,11 @@ func TestPGVectorLiteral(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPG_RequiresEnv runs a real PostgreSQL+pgvector round trip when DB_PG_URL
|
||||
// is set, and skips otherwise. There is no PG server in this environment by
|
||||
// default, so the pgvector-backed Add/Search path is validated end-to-end only
|
||||
// when the operator supplies a live database with the pgvector extension. The
|
||||
// SQLite path (the default) is fully covered by the sqlite_test.go suite.
|
||||
// TestPG_RequiresEnv 在设置了 DB_PG_URL 时跑一次真实的
|
||||
// PostgreSQL+pgvector 往返,否则跳过。本环境默认没有 PG
|
||||
// 服务,因此由 pgvector 支撑的 Add/Search 路径只有在
|
||||
// 操作者提供带 pgvector 扩展的活数据库时才端到端验证。
|
||||
// SQLite 路径(默认)由 sqlite_test.go 套件完整覆盖。
|
||||
func TestPG_RequiresEnv(t *testing.T) {
|
||||
url := os.Getenv("DB_PG_URL")
|
||||
if url == "" {
|
||||
@@ -81,7 +81,7 @@ func TestPG_RequiresEnv(t *testing.T) {
|
||||
t.Fatalf("pg a score = %v, want ~1.0", hits[0].Score)
|
||||
}
|
||||
|
||||
// The embedder is wired, so Index.Embed is usable.
|
||||
// embedder 已接入,因此 Index.Embed 可用。
|
||||
vs, err := idx.Embed(ctx, []string{"hello"})
|
||||
if err != nil {
|
||||
t.Fatalf("Embed: %v", err)
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
"git.yeij.top/AskaEth/Cyrene_v2/internal/runtime/embedding"
|
||||
)
|
||||
|
||||
// createVecItemsTable is the idempotent SQLite schema. vector is stored as a
|
||||
// BLOB of little-endian float32 values; dim records the dimension so the blob
|
||||
// can be decoded without external knowledge. The table is created by
|
||||
// NewSQLite; migration compatibility with internal/db is not required because
|
||||
// the vector store is self-contained (see internal/runtime/memory/store.go).
|
||||
// createVecItemsTable 是幂等的 SQLite schema。vector 以小端
|
||||
// float32 值的 BLOB 存储;dim 记录维度,因此 blob
|
||||
// 无需外部信息即可解码。该表由
|
||||
// NewSQLite 创建;不要求与 internal/db 的迁移兼容,因为
|
||||
// 向量存储是自包含的(见 internal/runtime/memory/store.go)。
|
||||
const createVecItemsTable = `
|
||||
CREATE TABLE IF NOT EXISTS vec_items (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS vec_items (
|
||||
dim INTEGER NOT NULL
|
||||
)`
|
||||
|
||||
// SQLiteIndex stores vectors as BLOBs in a SQLite vec_items table and answers
|
||||
// similarity with Go-side cosine similarity over a linear scan (milestone M4
|
||||
// semantics: small datasets, no ANN index).
|
||||
// SQLiteIndex 把向量以 BLOB 存入 SQLite 的 vec_items 表,并以
|
||||
// Go 侧线性扫描的余弦相似度作答相似度(M4 里程碑
|
||||
// 语义:小数据集,无 ANN 索引)。
|
||||
//
|
||||
// Concurrency is delegated to the *sql.DB, which is safe for concurrent use by
|
||||
// contract, so SQLiteIndex holds no further lock. Every query is parameterized
|
||||
// (?) and columns are listed explicitly.
|
||||
// 并发交由 *sql.DB,按约定它可安全并发使用,
|
||||
// 因此 SQLiteIndex 不再持锁。每个查询都参数化
|
||||
// (?)且列清单显式列出。
|
||||
type SQLiteIndex struct {
|
||||
db *sql.DB
|
||||
dim int
|
||||
@@ -39,14 +39,14 @@ type SQLiteIndex struct {
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewSQLite builds a SQLiteIndex on db for vectors of dimension dim, ensuring
|
||||
// the vec_items schema exists. db must be a live connection; a nil db returns
|
||||
// ErrNilDB, and a non-positive dim returns ErrInvalidDim. ctx controls schema
|
||||
// creation. log may be nil.
|
||||
// NewSQLite 在 db 上为维度 dim 的向量构建 SQLiteIndex,确保
|
||||
// vec_items schema 存在。db 必须是存活连接;nil db 返回
|
||||
// ErrNilDB,dim 非正返回 ErrInvalidDim。ctx 控制 schema
|
||||
// 创建。log 可为 nil。
|
||||
//
|
||||
// The returned index has no embedder wired in, so calling Embed directly
|
||||
// returns ErrNoEmbedder; use New to build a fully wired index, or wire one in
|
||||
// before calling Embed.
|
||||
// 返回的索引没有接入 embedder,因此直接调用 Embed
|
||||
// 会返回 ErrNoEmbedder;用 New 构建完全接好的索引,或在调用
|
||||
// Embed 前接入一个。
|
||||
func NewSQLite(ctx context.Context, db *sql.DB, dim int, log *slog.Logger) (*SQLiteIndex, error) {
|
||||
if db == nil {
|
||||
return nil, ErrNilDB
|
||||
@@ -62,16 +62,16 @@ func NewSQLite(ctx context.Context, db *sql.DB, dim int, log *slog.Logger) (*SQL
|
||||
return &SQLiteIndex{db: db, dim: dim, log: log}, nil
|
||||
}
|
||||
|
||||
// withEmbedder wires the vector-producing embedder into the index. It is used
|
||||
// by New; tests may also call it directly.
|
||||
// withEmbedder 把产向量的 embedder 接入索引。它由
|
||||
// New 使用;测试也可直接调用。
|
||||
func (s *SQLiteIndex) withEmbedder(e embedding.Embedder) { s.embedder = e }
|
||||
|
||||
// Dim returns the fixed vector dimension.
|
||||
// Dim 返回固定向量维度。
|
||||
func (s *SQLiteIndex) Dim() int { return s.dim }
|
||||
|
||||
// Embed turns texts into vectors with the wired embedder. It returns one Vector
|
||||
// per input text, aligned by index. With no embedder wired it returns
|
||||
// ErrNoEmbedder.
|
||||
// Embed 用接入的 embedder 把 texts 转成向量。每段输入文本
|
||||
// 对应一个按索引对齐的 Vector。未接入 embedder 时返回
|
||||
// ErrNoEmbedder。
|
||||
func (s *SQLiteIndex) Embed(ctx context.Context, texts []string) ([]Vector, error) {
|
||||
if s.embedder == nil {
|
||||
return nil, ErrNoEmbedder
|
||||
@@ -87,10 +87,10 @@ func (s *SQLiteIndex) Embed(ctx context.Context, texts []string) ([]Vector, erro
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Add persists the (id, vector) pair, upserting on a repeated id so the
|
||||
// latest vector wins. It rejects a blank id (ErrMissingID) and a vector whose
|
||||
// length differs from the index dimension (ErrDimMismatch). ctx aborts the
|
||||
// write when canceled.
|
||||
// Add 持久化 (id, vector) 对,id 重复时 upsert,
|
||||
// 最新的向量生效。它拒绝空 id(ErrMissingID)以及与
|
||||
// 索引维度不一致的向量(ErrDimMismatch)。ctx 被取消时
|
||||
// 中止写入。
|
||||
func (s *SQLiteIndex) Add(ctx context.Context, id string, v Vector) error {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return ErrMissingID
|
||||
@@ -108,10 +108,10 @@ func (s *SQLiteIndex) Add(ctx context.Context, id string, v Vector) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search returns up to topK ids most similar to q, ranked by descending cosine
|
||||
// similarity. It scans every persisted vector in Go (linear scan, no ANN). A
|
||||
// non-positive topK returns an empty result; a query whose length differs from
|
||||
// the index dimension returns ErrDimMismatch.
|
||||
// Search 返回与 q 最相似的最多 topK 个 id,按余弦相似度
|
||||
// 降序排列。它在 Go 侧扫描每个已持久化向量(线性扫描,无 ANN)。topK
|
||||
// 非正返回空结果;长度与
|
||||
// 索引维度不一致的查询返回 ErrDimMismatch。
|
||||
func (s *SQLiteIndex) Search(ctx context.Context, q Vector, topK int) ([]Hit, error) {
|
||||
if topK <= 0 {
|
||||
return []Hit{}, nil
|
||||
@@ -151,8 +151,8 @@ func (s *SQLiteIndex) Search(ctx context.Context, q Vector, topK int) ([]Hit, er
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying SQLite database. It is safe to call when the
|
||||
// database is already closed.
|
||||
// Close 释放底层的 SQLite 数据库。数据库
|
||||
// 已关闭时调用也是安全的。
|
||||
func (s *SQLiteIndex) Close() error {
|
||||
if s.db == nil {
|
||||
return nil
|
||||
@@ -164,7 +164,7 @@ func (s *SQLiteIndex) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeVector serializes v to a little-endian float32 BLOB. It cannot fail.
|
||||
// encodeVector 把 v 序列化为小端 float32 BLOB。它不会失败。
|
||||
func encodeVector(v Vector) []byte {
|
||||
buf := make([]byte, 4*len(v))
|
||||
for i, x := range v {
|
||||
@@ -173,9 +173,9 @@ func encodeVector(v Vector) []byte {
|
||||
return buf
|
||||
}
|
||||
|
||||
// decodeVector deserializes a BLOB back into a Vector. It rejects empty or
|
||||
// non-4-byte-aligned blobs so a corrupted row surfaces as an error rather than
|
||||
// a silent misread.
|
||||
// decodeVector 把 BLOB 反序列化回 Vector。它拒绝空或
|
||||
// 非 4 字节对齐的 blob,使损坏的行以错误暴露,而不是
|
||||
// 被静默误读。
|
||||
func decodeVector(blob []byte) (Vector, error) {
|
||||
if len(blob) == 0 || len(blob)%4 != 0 {
|
||||
return nil, fmt.Errorf("memory/vector: invalid vector blob length %d", len(blob))
|
||||
@@ -187,9 +187,9 @@ func decodeVector(blob []byte) (Vector, error) {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// cosine returns the cosine similarity of a and b in [-1, 1]. Zero-length or
|
||||
// mismatched vectors yield 0, so an inconsistent stored row never corrupts the
|
||||
// ranking.
|
||||
// cosine 返回 a 与 b 的余弦相似度,取值 [-1, 1]。长度为零或
|
||||
// 不一致的向量得 0,因此不一致的存储行绝不会破坏
|
||||
// 排名。
|
||||
func cosine(a, b Vector) float64 {
|
||||
if len(a) == 0 || len(b) == 0 || len(a) != len(b) {
|
||||
return 0
|
||||
|
||||
Reference in new issue
Block a user