Loading...
Loading...
Compare original and translation side by side
-- Enable extensions
CREATE EXTENSION IF NOT EXISTS vector;
CREATE EXTENSION IF NOT EXISTS pg_textsearch;
-- Table with both indexes
CREATE TABLE documents (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
content TEXT NOT NULL,
embedding halfvec(1536) NOT NULL
);
-- BM25 index for keyword search
CREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english');
-- HNSW index for semantic search
CREATE INDEX ON documents USING hnsw (embedding halfvec_cosine_ops);-- 启用扩展
CREATE EXTENSION IF NOT EXISTS vector;
CREATE EXTENSION IF NOT EXISTS pg_textsearch;
-- 创建包含双索引的表
CREATE TABLE documents (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
content TEXT NOT NULL,
embedding halfvec(1536) NOT NULL
);
-- 为关键词搜索创建BM25索引
CREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english');
-- 为语义搜索创建HNSW索引
CREATE INDEX ON documents USING hnsw (embedding halfvec_cosine_ops);<@>text_config'french''german'k1bCREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english', k1 = 1.5, b = 0.8);<@>text_config'french''german'k1bCREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english', k1 = 1.5, b = 0.8);1 / (k + rank)k-- Query 1: Keyword search (BM25)
-- $1: search text
SELECT id, content FROM documents ORDER BY content <@> $1 LIMIT 50;-- Query 2: Semantic search (separate query, run in parallel)
-- $1: embedding of your search text as halfvec(1536)
SELECT id, content FROM documents ORDER BY embedding <=> $1::halfvec(1536) LIMIT 50;undefined1 / (k + rank)k-- 查询1:关键词搜索(BM25)
-- $1:搜索文本
SELECT id, content FROM documents ORDER BY content <@> $1 LIMIT 50;-- 查询2:语义搜索(单独查询,并行执行)
-- $1:搜索文本的嵌入向量,格式为halfvec(1536)
SELECT id, content FROM documents ORDER BY embedding <=> $1::halfvec(1536) LIMIT 50;undefinedfor rank, row in enumerate(keyword_results, start=1):
scores[row['id']] = scores.get(row['id'], 0) + 1 / (k + rank)
content_map[row['id']] = row['content']
for rank, row in enumerate(semantic_results, start=1):
scores[row['id']] = scores.get(row['id'], 0) + 1 / (k + rank)
content_map[row['id']] = row['content']
sorted_ids = sorted(scores, key=scores.get, reverse=True)[:limit]
return [{'id': id, 'content': content_map[id], 'score': scores[id]} for id in sorted_ids]
```typescript
// Client-side RRF fusion (TypeScript)
type Row = { id: number; content: string };
type Result = Row & { score: number };
function rrfFusion(keywordResults: Row[], semanticResults: Row[], k = 60, limit = 10): Result[] {
const scores = new Map<number, number>();
const contentMap = new Map<number, string>();
keywordResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (k + i + 1));
contentMap.set(row.id, row.content);
});
semanticResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (k + i + 1));
contentMap.set(row.id, row.content);
});
return [...scores.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, limit)
.map(([id, score]) => ({ id, content: contentMap.get(id)!, score }));
}for rank, row in enumerate(keyword_results, start=1):
scores[row['id']] = scores.get(row['id'], 0) + 1 / (k + rank)
content_map[row['id']] = row['content']
for rank, row in enumerate(semantic_results, start=1):
scores[row['id']] = scores.get(row['id'], 0) + 1 / (k + rank)
content_map[row['id']] = row['content']
sorted_ids = sorted(scores, key=scores.get, reverse=True)[:limit]
return [{'id': id, 'content': content_map[id], 'score': scores[id]} for id in sorted_ids]
```typescript
// 客户端RRF融合(TypeScript)
type Row = { id: number; content: string };
type Result = Row & { score: number };
function rrfFusion(keywordResults: Row[], semanticResults: Row[], k = 60, limit = 10): Result[] {
const scores = new Map<number, number>();
const contentMap = new Map<number, string>();
keywordResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (k + i + 1));
contentMap.set(row.id, row.content);
});
semanticResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + 1 / (k + i + 1));
contentMap.set(row.id, row.content);
});
return [...scores.entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, limit)
.map(([id, score]) => ({ id, content: contentMap.get(id)!, score }));
}| Parameter | Default | Description |
|---|---|---|
| 60 | Smoothing constant. Higher values reduce rank differences; 60 is standard |
| Candidates per search | 50 | Higher = better recall, more work |
| Final limit | 10 | Results returned after fusion |
| 参数 | 默认值 | 描述 |
|---|---|---|
| 60 | 平滑常数,值越大排名差异越小,60为标准取值 |
| 单方法候选数 | 50 | 数值越高召回率越好,但计算量越大 |
| 最终返回数量 | 10 | 融合后返回的结果数量 |
undefinedundefined
```typescript
// Weight semantic search 2x higher than keyword
const keywordWeight = 1.0;
const semanticWeight = 2.0;
keywordResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + keywordWeight / (k + i + 1));
});
semanticResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + semanticWeight / (k + i + 1));
});
```typescript
// 语义搜索权重为关键词搜索的2倍
const keywordWeight = 1.0;
const semanticWeight = 2.0;
keywordResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + keywordWeight / (k + i + 1));
});
semanticResults.forEach((row, i) => {
scores.set(row.id, (scores.get(row.id) ?? 0) + semanticWeight / (k + i + 1));
});cross-encoder/ms-marco-MiniLM-L-6-v2undefinedcross-encoder/ms-marco-MiniLM-L-6-v2undefined
```typescript
import { CohereClientV2 } from 'cohere-ai';
// 1. Fuse results with RRF (more candidates for reranking)
const candidates = rrfFusion(keywordResults, semanticResults, 60, 100);
// 2. Rerank via API (example uses Cohere SDK; Jina, Voyage, and others work similarly)
const cohere = new CohereClientV2({ token: COHERE_API_KEY });
const reranked = await cohere.rerank({
model: 'rerank-v3.5',
query: queryText,
documents: candidates.map(c => c.content),
topN: 10
});
// 3. Map back to original documents
const results = reranked.results.map(r => candidates[r.index]);
```typescript
import { CohereClientV2 } from 'cohere-ai';
// 1. 使用RRF融合结果(保留更多候选用于重排序)
const candidates = rrfFusion(keywordResults, semanticResults, 60, 100);
// 2. 通过API进行重排序(示例使用Cohere SDK;Jina、Voyage等平台也支持类似功能)
const cohere = new CohereClientV2({ token: COHERE_API_KEY });
const reranked = await cohere.rerank({
model: 'rerank-v3.5',
query: queryText,
documents: candidates.map(c => c.content),
topN: 10
});
// 3. 映射回原始文档
const results = reranked.results.map(r => candidates[r.index]);smallint[]-- Enable pgvectorscale (in addition to pgvector)
CREATE EXTENSION IF NOT EXISTS vectorscale;
-- Table with label column for filtering
CREATE TABLE documents (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
content TEXT NOT NULL,
embedding halfvec(1536) NOT NULL,
labels smallint[] NOT NULL -- e.g., category IDs, tenant IDs
);
-- StreamingDiskANN index with label filtering
CREATE INDEX ON documents USING diskann (embedding vector_cosine_ops, labels);
-- BM25 index for keyword search
CREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english');
-- Filtered semantic search using && (array overlap)
SELECT id, content FROM documents
WHERE labels && ARRAY[1, 3]::smallint[]
ORDER BY embedding <=> $1::halfvec(1536) LIMIT 50;smallint[]-- 启用pgvectorscale(需同时启用pgvector)
CREATE EXTENSION IF NOT EXISTS vectorscale;
-- 创建带标签列的表用于过滤
CREATE TABLE documents (
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
content TEXT NOT NULL,
embedding halfvec(1536) NOT NULL,
labels smallint[] NOT NULL -- 例如类别ID、租户ID
);
-- 创建带标签过滤的StreamingDiskANN索引
CREATE INDEX ON documents USING diskann (embedding vector_cosine_ops, labels);
-- 为关键词搜索创建BM25索引
CREATE INDEX ON documents USING bm25 (content) WITH (text_config = 'english');
-- 使用&&(数组重叠)进行过滤语义搜索
SELECT id, content FROM documents
WHERE labels && ARRAY[1, 3]::smallint[]
ORDER BY embedding <=> $1::halfvec(1536) LIMIT 50;-- Force index usage for verification (planner may prefer seqscan on small tables)
SET enable_seqscan = off;
-- Verify BM25 index is used
EXPLAIN SELECT id, content FROM documents ORDER BY content <@> 'search text' LIMIT 10;
-- Look for: Index Scan using ... (bm25)
-- Verify HNSW index is used
EXPLAIN SELECT id, content FROM documents ORDER BY embedding <=> '[0.1, 0.2, ...]'::halfvec(1536) LIMIT 10;
-- Look for: Index Scan using ... (hnsw)
SET enable_seqscan = on; -- Re-enable for normal operation
-- Check index sizes
SELECT indexname, pg_size_pretty(pg_relation_size(indexname::regclass)) AS size
FROM pg_indexes WHERE tablename = 'documents';enable_seqscan = off<@><=>-- 强制使用索引以验证(对于小表,查询优化器可能更倾向于顺序扫描)
SET enable_seqscan = off;
-- 验证BM25索引是否被使用
EXPLAIN SELECT id, content FROM documents ORDER BY content <@> 'search text' LIMIT 10;
-- 需包含:Index Scan using ... (bm25)
-- 验证HNSW索引是否被使用
EXPLAIN SELECT id, content FROM documents ORDER BY embedding <=> '[0.1, 0.2, ...]'::halfvec(1536) LIMIT 10;
-- 需包含:Index Scan using ... (hnsw)
SET enable_seqscan = on; -- 恢复正常操作
-- 检查索引大小
SELECT indexname, pg_size_pretty(pg_relation_size(indexname::regclass)) AS size
FROM pg_indexes WHERE tablename = 'documents';enable_seqscan = off<@><=>| Symptom | Likely Cause | Fix |
|---|---|---|
| Missing exact matches | Keyword search not returning them | Check BM25 index exists; verify text_config matches content language |
| Poor semantic results | Embedding model mismatch | Ensure query embedding uses same model as stored embeddings |
| Slow queries | Large candidate pools or missing indexes | Reduce inner LIMIT; verify both indexes exist and are used (EXPLAIN) |
| Skewed results | One method dominating | Adjust RRF weights; verify both searches return reasonable candidates |
| 症状 | 可能原因 | 解决方法 |
|---|---|---|
| 缺失精确匹配结果 | 关键词搜索未返回该结果 | 检查BM25索引是否存在;验证text_config是否与内容语言匹配 |
| 语义搜索结果质量差 | 嵌入模型不匹配 | 确保查询嵌入向量与存储的嵌入向量使用相同模型 |
| 查询速度慢 | 候选集过大或缺失索引 | 减小内部LIMIT值;验证两个索引是否存在并被使用(通过EXPLAIN) |
| 结果倾斜 | 某一种搜索方法占主导 | 调整RRF权重;验证两种搜索方法均返回合理的候选结果 |