AskaEth
41f653b672
- 所有对外称呼从 QQ 改为 OBv11（注释/提示词/日志/配置项） - 新增 PlatformFormat 结构体，统一管理平台消息标记格式 - defaultPlatformFormats() 注册表替代硬编码 qqTargetRe - extractProactiveMessage 改为 Thinker 方法，遍历格式注册表匹配 - 配置项重命名: QQ_BOT_PORT → OBV11_BOT_PORT, QQBotPort → OBv11BotPort - 标记格式: 【QQ群聊】→【OBv11群聊】、【QQ私聊】→【OBv11私聊】 Co-Authored-By: Claude <noreply@anthropic.com>
231 lines
10 KiB
Python
231 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
昔涟声纹搜索 — 多线程 + 全精度 pyin
|
||
用法: python search_cyrene.py
|
||
监控: tail -f cyrene_confirmed/search.log
|
||
"""
|
||
import os, sys, shutil, json, time, warnings, logging
|
||
from multiprocessing import Pool, cpu_count
|
||
import numpy as np
|
||
import librosa
|
||
warnings.filterwarnings('ignore')
|
||
|
||
# ═══════════ 配置 ═══════════
|
||
SEARCH_DIR = r"D:\Project\Code\Uni\Cyrene-Voice-Model\data\cleaned"
|
||
OUT_DIR = r"D:\Project\Code\Uni\Cyrene-Voice-Model\data\cyrene_confirmed"
|
||
WORKERS = max(1, cpu_count() - 1)
|
||
BATCH = 500 # 每批文件数
|
||
|
||
CONFIRMED = [
|
||
# ── tier1 确认 (22) ──
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0358_02ff22a9.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0369_0315d90a.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0370_0317b5bf.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0391_033cbeea.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0392_033d9e2e.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0398_0346fa30.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0417_0367cbc5.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0330_02cb864d.wav",
|
||
"External_del_3.5_chapter_2/External_del_3.5_chapter_2_0357_02fd4ab1.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0106_009dfe25.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0107_009fba53.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0112_00a89eb2.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0124_00c08c68.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0128_00c6ba19.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0365_026cf0de.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0371_0278f79f.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0374_02803ab5.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0409_02bcd547.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0410_02bec9f4.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0437_02e57ff0.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0440_02e7d98f.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0467_031daa9e.wav",
|
||
# ── tier2 新确认 (31) ──
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0102_0095ba46.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0109_00a458cc.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0126_00c44990.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0130_00c95298.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0359_02670f55.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0364_026b8023.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0366_026f3922.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0369_0274d62a.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0372_027b67ef.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0392_029e92a6.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0413_02c1a418.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0415_02c466f1.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0417_02c8ffc0.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0430_02dbad24.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0431_02dda165.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0434_02e216cf.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0443_02eb0bc4.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0455_03048430.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0462_0316bcd9.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0463_03181c56.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0464_03195142.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0468_031e8bd6.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0469_03208c8a.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0484_03339e50.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0488_033a7664.wav",
|
||
"External_del_3.4_chapter_0/External_del_3.4_chapter_0_0489_033b5c44.wav",
|
||
]
|
||
|
||
# ═══════════ 纯函数 (供 Pool 调用) ═══════════
|
||
|
||
def extract_features(wav_path):
|
||
"""全精度 pyin + MFCC20 + delta×2 + spectral"""
|
||
try:
|
||
y, sr = librosa.load(wav_path, sr=22050, mono=True)
|
||
if len(y) < sr * 0.25:
|
||
return None
|
||
f0, _, _ = librosa.pyin(y, fmin=80, fmax=600, sr=sr)
|
||
f0 = f0[~np.isnan(f0)]
|
||
if len(f0) < 10:
|
||
return None
|
||
mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=20)
|
||
mfcc_d = librosa.feature.delta(mfcc)
|
||
mfcc_d2 = librosa.feature.delta(mfcc, order=2)
|
||
cent = librosa.feature.spectral_centroid(y=y, sr=sr)
|
||
roll = librosa.feature.spectral_rolloff(y=y, sr=sr)
|
||
return np.concatenate([
|
||
[np.mean(f0), np.std(f0), np.percentile(f0,10), np.percentile(f0,25),
|
||
np.percentile(f0,50), np.percentile(f0,75), np.percentile(f0,90)],
|
||
np.mean(mfcc,axis=1), np.std(mfcc,axis=1),
|
||
np.mean(mfcc_d,axis=1), np.std(mfcc_d,axis=1),
|
||
np.mean(mfcc_d2,axis=1), np.std(mfcc_d2,axis=1),
|
||
[np.mean(cent), np.std(cent), np.mean(roll), np.std(roll)],
|
||
]).astype(np.float64)
|
||
except:
|
||
return None
|
||
|
||
# ═══════════ 主流程 ═══════════
|
||
|
||
def main():
|
||
os.makedirs(OUT_DIR, exist_ok=True)
|
||
LOG_FILE = os.path.join(OUT_DIR, "search.log")
|
||
|
||
# 日志: 文件 + 终端
|
||
log = logging.getLogger("cyrene")
|
||
log.setLevel(logging.INFO)
|
||
for h in [logging.FileHandler(LOG_FILE, encoding='utf-8'), logging.StreamHandler(sys.stdout)]:
|
||
h.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", "%H:%M:%S"))
|
||
log.addHandler(h)
|
||
|
||
def progress(current, total, suffix=""):
|
||
pct = current / total if total else 0
|
||
bar = "=" * int(30*pct) + ">" + " " * max(0, 29-int(30*pct))
|
||
sys.stdout.write(f"\r [{bar}] {pct*100:5.1f}% {current}/{total} {suffix}")
|
||
sys.stdout.flush()
|
||
|
||
log.info("=" * 55)
|
||
log.info("昔涟声纹搜索 — 多线程全精度模式")
|
||
log.info(f" 线程: {WORKERS} | 批量: {BATCH}")
|
||
log.info("=" * 55)
|
||
|
||
t_start = time.time()
|
||
|
||
# ── Step 1: 参考模板 ──
|
||
log.info("\n[1/3] 提取参考模板...")
|
||
ref_feats = []
|
||
for fname in CONFIRMED:
|
||
path = os.path.join(SEARCH_DIR, fname)
|
||
f = extract_features(path)
|
||
if f is not None:
|
||
ref_feats.append(f)
|
||
log.info(f" OK {os.path.basename(fname)}")
|
||
template = np.mean(ref_feats, axis=0)
|
||
log.info(f" 模板: {len(ref_feats)} 文件 | pitch={template[0]:.0f}Hz | dim={len(template)}")
|
||
|
||
# ── Step 2: 文件列表 ──
|
||
log.info(f"\n[2/3] 收集文件...")
|
||
all_wavs = []
|
||
for root, dirs, files in os.walk(SEARCH_DIR):
|
||
for f in files:
|
||
if f.endswith(".wav"):
|
||
all_wavs.append(os.path.join(root, f))
|
||
log.info(f" {len(all_wavs):,} 个 WAV 文件")
|
||
|
||
# ── Step 3: 多线程搜索 ──
|
||
log.info(f"\n[3/3] 多线程搜索 ({len(all_wavs):,} 文件, {WORKERS} 线程)...")
|
||
log.info("-" * 55)
|
||
t2 = time.time()
|
||
results = []
|
||
errors = 0
|
||
|
||
pool = Pool(WORKERS)
|
||
done = 0
|
||
|
||
for feats in pool.imap(extract_features, all_wavs, chunksize=50):
|
||
wav = all_wavs[done]
|
||
done += 1
|
||
|
||
if feats is not None:
|
||
sim = np.dot(feats, template) / (np.linalg.norm(feats) * np.linalg.norm(template) + 1e-8)
|
||
p = feats[0]
|
||
penalty = 1.0 / (1.0 + abs(p - template[0]) / 100)
|
||
results.append((sim * 0.6 + penalty * 0.4, sim, p, wav))
|
||
else:
|
||
errors += 1
|
||
|
||
# 进度更新
|
||
if done % 100 == 0 or done == len(all_wavs):
|
||
elapsed = time.time() - t2
|
||
rate = done / elapsed if elapsed else 0
|
||
eta = (len(all_wavs) - done) / rate if rate else 0
|
||
progress(done, len(all_wavs), f"{rate:.0f}f/s ETA{eta:.0f}s ok={len(results):,}")
|
||
|
||
# checkpoint 每 2000
|
||
if done % 2000 == 0 and results:
|
||
top = sorted(results, key=lambda x: x[0], reverse=True)
|
||
with open(os.path.join(OUT_DIR, f"ckpt_{done}.json"), 'w') as jf:
|
||
json.dump([(float(x[0]), float(x[2]), x[3]) for x in top[:500]], jf)
|
||
|
||
pool.close()
|
||
pool.join()
|
||
|
||
print()
|
||
log.info(f" 完成: {len(results):,} ok | {errors} skip | {time.time()-t2:.0f}s ({len(results)/(time.time()-t2):.0f} f/s)")
|
||
log.info(f" 总耗时: {time.time()-t_start:.0f}s")
|
||
|
||
# ── 排序 ──
|
||
results.sort(key=lambda x: x[0], reverse=True)
|
||
|
||
log.info(f"\n{'='*55}")
|
||
log.info(f"Top 50 候选 (★ = combo > 0.85)")
|
||
log.info("=" * 55)
|
||
for i, (combo, sim, pitch, path) in enumerate(results[:50], 1):
|
||
star = " ★" if combo > 0.85 else ""
|
||
d = os.path.basename(os.path.dirname(path))
|
||
f = os.path.basename(path)
|
||
log.info(f" {i:2d}. [{combo:.4f}]{star} {d}/{f}")
|
||
|
||
# ── 来源统计 ──
|
||
log.info(f"\n来源分布 (combo > 0.85):")
|
||
srcs = {}
|
||
for combo, sim, pitch, path in results:
|
||
if combo > 0.85:
|
||
d = os.path.basename(os.path.dirname(path))
|
||
srcs[d] = srcs.get(d, 0) + 1
|
||
for d in sorted(srcs):
|
||
log.info(f" {d}: {srcs[d]} 文件")
|
||
|
||
# ── 导出 ──
|
||
copied = 0
|
||
for combo, sim, pitch, path in results:
|
||
if combo > 0.85:
|
||
dst = os.path.join(OUT_DIR, os.path.basename(path))
|
||
if not os.path.exists(dst):
|
||
shutil.copy2(path, dst)
|
||
copied += 1
|
||
log.info(f"\n导出: {copied} 文件 → {OUT_DIR}")
|
||
|
||
rp = os.path.join(OUT_DIR, "search_results.json")
|
||
with open(rp, 'w') as f:
|
||
json.dump([(float(s), float(p), w) for s, p, w in results], f, ensure_ascii=False)
|
||
log.info(f"结果: {rp}")
|
||
log.info(f"日志: {LOG_FILE}")
|
||
log.info(f"\n{'='*55}")
|
||
log.info("DONE")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|