35 lines
1.6 KiB
Go
35 lines
1.6 KiB
Go
// Package embedding provides text-to-vector embedding and keyword full-text
|
|
// search for the Cyrene runtime.
|
|
//
|
|
// It implements the vectorization path described in DevDocs/design/04-运行时.md
|
|
// §7: an Embedder interface with two implementations — a dependency-free
|
|
// n-gram hashing embedder (the offline default) and an optional OpenAI-
|
|
// compatible /embeddings API embedder — plus a keyword full-text search (FTS)
|
|
// inverted index.
|
|
//
|
|
// The runtime has two coexisting retrieval levels: the n-gram/API vectors are
|
|
// semantic-level, while the FTS inverted index is keyword-level. Persistent
|
|
// vector storage (SQLite vector extension / pgvector) is deliberately out of
|
|
// scope for this package and belongs to milestone M4 (see
|
|
// DevDocs/开发时间线.md); M1 only performs embedding and indexes for search.
|
|
package embedding
|
|
|
|
import "context"
|
|
|
|
// Vector is a dense embedding: one float32 per dimension.
|
|
type Vector []float32
|
|
|
|
// Embedder converts a batch of texts into dense vectors of a fixed dimension.
|
|
//
|
|
// Implementations must be deterministic for identical input and safe for
|
|
// concurrent use. Embed returns one Vector per input text, aligned by index;
|
|
// every returned vector has Dim() elements. ctx controls cancellation and must
|
|
// be honored: a canceled context returns immediately with a non-nil error.
|
|
type Embedder interface {
|
|
// Embed returns one vector per input text, in the same order as texts.
|
|
Embed(ctx context.Context, texts []string) ([]Vector, error)
|
|
// Dim returns the fixed number of dimensions in every produced vector. It
|
|
// must match the length of each vector returned by Embed.
|
|
Dim() int
|
|
}
|