feat(transcription): detect the session language and transcribe PCM b…

…uffers Transformers.js 4.2.0 has no language detection: with no language token it warns and transcribes as English, so Chinese voice messages came back as nonsense. Resolve the language from the session's own text messages before the audio reaches Whisper, and add the single-attachment PCM entry point the desktop and CLI Web runtimes need once they decode audio in the browser. - core: detectSessionLanguage / resolveTranscriptionLanguage (CJK vs Latin character ratio over the 300 most recent text messages) - node-runtime: transcribeAttachmentPcm, with a 30-minute PCM ceiling and a coded error for non-audio attachments; transcribePcm no longer accepts auto - config: [transcription] model / language, read as the CLI flag defaults Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Zycheng114514 · Sep 8, 2026 · Sep 8, 2026 · Sep 8, 2026 · Sep 8, 2026 · Sep 8, 2026
commit 4a6de9c754bac63ebaa1c8c1cc5c2c5a0f6687bd
diff --git a/apps/cli/src/transcribe/command.ts b/apps/cli/src/transcribe/command.ts
@@ -8,6 +8,7 @@
 
 import path from 'node:path'
 import type { Command } from 'commander'
+import { resolveTranscriptionLanguage } from '@openchatlab/core'
 import {
   SemanticIndexConfigStore,
   SEMANTIC_INDEX_CONFIG_FILE,
@@ -21,6 +22,7 @@ import {
   type TranscriptionProfileId,
   type TranscriptionSkip,
 } from '@openchatlab/node-runtime'
+import { loadConfig } from '@openchatlab/config'
 import { initRuntime } from '../runtime'
 import { resolveCliLocalEmbeddingRuntimeConfig } from '../semantic-index/local-runtime'
 
@@ -33,8 +35,8 @@ const SKIP_REASON_TEXT: Record<TranscriptionSkip['reason'], string> = {
 
 interface TranscribeCommandOptions {
   session: string
-  model: string
-  language: string
+  model?: string
+  language?: string
   attachment?: string[]
   dryRun?: boolean
 }
@@ -44,8 +46,11 @@ export function registerTranscribeCommand(program: Command): void {
     .command('transcribe')
     .description('Transcribe the voice attachments of a session with a local Whisper model')
     .requiredOption('--session <id>', 'Session ID whose voice attachments should be transcribed')
-    .option(`--model <${TRANSCRIPTION_PROFILE_IDS.join('|')}>`, 'Whisper model size', 'base')
-    .option(`--language <${LANGUAGES.join('|')}>`, 'Spoken language; auto leaves it to the model', 'auto')
+    .option(`--model <${TRANSCRIPTION_PROFILE_IDS.join('|')}>`, 'Whisper model size (default: [transcription] model)')
+    .option(
+      `--language <${LANGUAGES.join('|')}>`,
+      'Spoken language; auto reads it off the session (default: [transcription] language)'
+    )
     .option('--attachment <id...>', 'Restrict the run to these attachment IDs')
     .option('--dry-run', 'List what would be transcribed and why the rest is skipped')
     .action(async (options: TranscribeCommandOptions) => {
@@ -59,8 +64,10 @@ export function registerTranscribeCommand(program: Command): void {
 }
 
 async function runTranscribe(options: TranscribeCommandOptions): Promise<void> {
-  const profile = parseProfile(options.model)
-  const language = parseLanguage(options.language)
+  // Unset flags fall back to the shared setting the desktop app and CLI Web write.
+  const configured = loadConfig().transcription
+  const profile = parseProfile(options.model ?? configured.model)
+  const language = parseLanguage(options.language ?? configured.language)
   const attachmentIds = parseAttachmentIds(options.attachment)
 
   const { pathProvider, dbManager } = initRuntime()
@@ -69,6 +76,8 @@ async function runTranscribe(options: TranscribeCommandOptions): Promise<void> {
     const db = dbManager.openWritable(options.session)
     if (!db) throw new Error(`Session ${options.session} not found`)
 
+    // Resolve `auto` here so the summary line names the language Whisper is told.
+    const resolvedLanguage = resolveTranscriptionLanguage(db, language)
     const plan = planSessionTranscription(db, attachmentIds)
     printSkipped(plan.skipped)
 
@@ -95,14 +104,14 @@ async function runTranscribe(options: TranscribeCommandOptions): Promise<void> {
       profile,
     })
 
-    console.log(`Transcribing ${plan.pending.length} attachment(s) with ${transcriber.modelId} (${language})`)
+    console.log(`Transcribing ${plan.pending.length} attachment(s) with ${transcriber.modelId} (${resolvedLanguage})`)
     try {
       const result = await transcribeSessionAttachments({
         db,
         transcriber,
         sessionId: options.session,
         attachmentIds,
-        language,
+        language: resolvedLanguage,
         // stderr: stdout stays a clean summary for agent/script consumers.
         onProgress: (progress) =>
           console.error(`  [${progress.completed}/${progress.total}] #${progress.attachmentId} ${progress.status}`),

diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts
@@ -7,7 +7,7 @@
 export { loadConfig, getConfigPath, getConfigDir, writeConfigField } from './loader'
 export { setConfigField, ConfigSetError } from './set-config-field'
 export type { ConfigSetResult, ConfigSetErrorReason } from './set-config-field'
-export { configSchema, desktopConfigSchema, DEFAULT_API_PORT } from './schema'
+export { configSchema, desktopConfigSchema, transcriptionConfigSchema, DEFAULT_API_PORT } from './schema'
 export type {
   ChatLabConfig,
   LlmConfig,
@@ -17,6 +17,7 @@ export type {
   UiConfig,
   CliConfig,
   DesktopConfig,
+  TranscriptionConfig,
 } from './schema'
 export {
   loadAuthProfiles,

diff --git a/packages/config/src/schema.test.ts b/packages/config/src/schema.test.ts
@@ -51,3 +51,22 @@ test('accepts supported desktop UI scales and rejects out-of-range or non-numeri
     assert.throws(() => configSchema.parse({ desktop: { ui_scale: uiScale } }))
   }
 })
+
+test('defaults transcription to the base model and automatic language', () => {
+  const config = configSchema.parse({})
+
+  assert.equal(config.transcription.model, 'base')
+  assert.equal(config.transcription.language, 'auto')
+})
+
+test('accepts supported transcription models and languages and rejects the rest', () => {
+  for (const model of ['tiny', 'base', 'small']) {
+    assert.equal(configSchema.parse({ transcription: { model } }).transcription.model, model)
+  }
+  for (const language of ['auto', 'zh', 'en']) {
+    assert.equal(configSchema.parse({ transcription: { language } }).transcription.language, language)
+  }
+
+  assert.throws(() => configSchema.parse({ transcription: { model: 'large' } }))
+  assert.throws(() => configSchema.parse({ transcription: { language: 'ja' } }))
+})
diff --git a/packages/config/src/schema.ts b/packages/config/src/schema.ts
@@ -51,6 +51,13 @@ export const desktopConfigSchema = z.object({
   ui_scale: z.number().min(0.8).max(2).default(1),
 })
 
+export const transcriptionConfigSchema = z.object({
+  /** Whisper size; base is the quality/download compromise (~80 MB). */
+  model: z.enum(['tiny', 'base', 'small']).default('base'),
+  /** `auto` picks zh or en from the session's own messages. */
+  language: z.enum(['auto', 'zh', 'en']).default('auto'),
+})
+
 export const configSchema = z.object({
   llm: llmConfigSchema.default({}),
   data: dataConfigSchema.default({}),
@@ -59,6 +66,7 @@ export const configSchema = z.object({
   ui: uiConfigSchema.default({}),
   cli: cliConfigSchema.default({}),
   desktop: desktopConfigSchema.default({}),
+  transcription: transcriptionConfigSchema.default({}),
 })
 
 export type ChatLabConfig = z.infer<typeof configSchema>
@@ -69,3 +77,4 @@ export type LocaleConfig = z.infer<typeof localeConfigSchema>
 export type UiConfig = z.infer<typeof uiConfigSchema>
 export type CliConfig = z.infer<typeof cliConfigSchema>
 export type DesktopConfig = z.infer<typeof desktopConfigSchema>
+export type TranscriptionConfig = z.infer<typeof transcriptionConfigSchema>
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
@@ -151,6 +151,8 @@ export {
   applyTranscript,
   formatVoiceTranscription,
   listPendingAudioAttachments,
+  detectSessionLanguage,
+  resolveTranscriptionLanguage,
 } from './query'
 
 // 查询类型
@@ -278,6 +280,8 @@ export type {
   ApplyTranscriptInput,
   ApplyTranscriptResult,
   PendingAudioAttachment,
+  ResolvedTranscriptionLanguage,
+  TranscriptionLanguage,
 } from './query'
 
 // 消息类型语义映射

diff --git a/packages/core/src/query/index.ts b/packages/core/src/query/index.ts
@@ -6,6 +6,9 @@ export type { MessageAttachment } from './message-attachments'
 export { applyTranscript, formatVoiceTranscription, listPendingAudioAttachments } from './attachment-transcripts'
 export type { ApplyTranscriptInput, ApplyTranscriptResult, PendingAudioAttachment } from './attachment-transcripts'
 
+export { detectSessionLanguage, resolveTranscriptionLanguage } from './session-language'
+export type { ResolvedTranscriptionLanguage, TranscriptionLanguage } from './session-language'
+
 export {
   isChatSessionDb,
   getSessionMeta,

diff --git a/packages/core/src/query/session-language.test.ts b/packages/core/src/query/session-language.test.ts
@@ -0,0 +1,91 @@
+/**
+ * Session language detection.
+ *
+ * Guards transcription quality: Whisper transcribes a Chinese voice message as
+ * garbled English when it is told the wrong language, and Transformers.js has no
+ * detection of its own, so this heuristic is the only thing standing between a
+ * Chinese chat and unusable transcripts.
+ */
+
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { CHAT_DB_SCHEMA } from '../schema'
+import { SqliteTestAdapter } from './__tests__/sqlite-test-adapter'
+import { openTestSqliteDatabase } from '../../../../tests/helpers/sqlite.mts'
+import { detectSessionLanguage, resolveTranscriptionLanguage } from './session-language'
+
+interface MessageSeed {
+  content: string
+  /** MessageType.TEXT unless the case is about non-text rows being ignored. */
+  type?: number
+}
+
+function createSessionDb(seeds: MessageSeed[]) {
+  const raw = openTestSqliteDatabase()
+  raw.exec(CHAT_DB_SCHEMA)
+  raw.prepare(`INSERT INTO member (platform_id, account_name) VALUES ('u1', 'Alice')`).run()
+  const insert = raw.prepare(`INSERT INTO message (sender_id, ts, type, content) VALUES (1, ?, ?, ?)`)
+  for (const [index, seed] of seeds.entries()) {
+    insert.run(index + 1, seed.type ?? 0, seed.content)
+  }
+  return { raw, db: new SqliteTestAdapter(raw) }
+}
+
+const cases: Array<{ name: string; seeds: MessageSeed[]; expected: 'zh' | 'en' }> = [
+  {
+    name: 'a Chinese chat that also uses English words',
+    seeds: [
+      { content: '今天的 deadline 是几点？' },
+      { content: '我下午把 PR 发出来，你 review 一下' },
+      { content: '好的，谢谢' },
+    ],
+    expected: 'zh',
+  },
+  {
+    name: 'an English-only chat',
+    seeds: [
+      { content: 'are we still on for tomorrow?' },
+      { content: 'yes, same time as last week' },
+      { content: 'great, see you then' },
+    ],
+    expected: 'en',
+  },
+  { name: 'a session with no messages', seeds: [], expected: 'en' },
+  {
+    name: 'a session whose only text is punctuation and numbers',
+    seeds: [{ content: '123' }, { content: '???' }],
+    expected: 'en',
+  },
+  {
+    name: 'a session whose Chinese lives in non-text messages only',
+    seeds: [
+      { content: 'ok sounds good' },
+      // Voice/system rows are not evidence of what the members type.
+      { content: '这是一条系统提示消息，内容全是中文', type: 80 },
+    ],
+    expected: 'en',
+  },
+]
+
+for (const { name, seeds, expected } of cases) {
+  test(`detectSessionLanguage returns ${expected} for ${name}`, () => {
+    const { raw, db } = createSessionDb(seeds)
+    try {
+      assert.equal(detectSessionLanguage(db), expected)
+    } finally {
+      raw.close()
+    }
+  })
+}
+
+test('resolveTranscriptionLanguage only detects for auto', () => {
+  const { raw, db } = createSessionDb([{ content: '我们晚点再聊' }])
+  try {
+    assert.equal(resolveTranscriptionLanguage(db, 'auto'), 'zh')
+    // An explicit choice wins even when it contradicts the messages.
+    assert.equal(resolveTranscriptionLanguage(db, 'en'), 'en')
+    assert.equal(resolveTranscriptionLanguage(db, 'zh'), 'zh')
+  } finally {
+    raw.close()
+  }
+})
diff --git a/packages/core/src/query/session-language.ts b/packages/core/src/query/session-language.ts
@@ -0,0 +1,82 @@
+/**
+ * Which language a session is spoken in, used to pick Whisper's language token.
+ *
+ * Transformers.js 4.2.0 has no language detection: with no language token it
+ * warns and transcribes as English, so a Chinese voice message comes back as
+ * nonsense English. The caller has to decide before the audio reaches the model,
+ * and the only signal available without listening to it is the text already in
+ * the chat — a good one, because a chat log is written in the language its
+ * participants speak.
+ *
+ * Deliberately a character-ratio count and not word segmentation: the answer is
+ * a two-way choice on hundreds of messages, and the two scripts do not overlap.
+ */
+
+import { MessageType } from '@openchatlab/shared-types'
+import type { DatabaseAdapter } from '../interfaces'
+import { hasTable } from './filters'
+
+/** What the user asked for; `auto` is resolved against the session's messages. */
+export type TranscriptionLanguage = 'auto' | 'zh' | 'en'
+
+/** What Whisper is actually told. */
+export type ResolvedTranscriptionLanguage = 'zh' | 'en'
+
+/** Messages sampled for the decision; recent ones describe how the chat reads today. */
+const SAMPLE_SIZE = 300
+
+/**
+ * Chinese, Japanese and Korean text runs far fewer characters than the Latin
+ * transcription of the same speech, so a fifth of the characters being CJK
+ * already means the chat is not an English one.
+ */
+const CJK_RATIO_THRESHOLD = 0.2
+
+const CJK_CHAR_REGEX = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/gu
+const LATIN_LETTER_REGEX = /\p{Script=Latin}/gu
+
+/**
+ * Guess the session's spoken language from its text messages.
+ *
+ * Falls back to `en` for a session with no text at all — that is what Whisper
+ * would have done anyway, and it keeps an empty session from failing.
+ */
+export function detectSessionLanguage(db: DatabaseAdapter): ResolvedTranscriptionLanguage {
+  if (!hasTable(db, 'message')) return 'en'
+
+  const rows = db
+    .prepare(
+      `SELECT content
+         FROM message
+        WHERE type = ${MessageType.TEXT} AND content IS NOT NULL AND TRIM(content) <> ''
+        ORDER BY ts DESC
+        LIMIT ${SAMPLE_SIZE}`
+    )
+    .all() as unknown as Array<{ content: string }>
+
+  let cjk = 0
+  let latin = 0
+  for (const row of rows) {
+    cjk += countMatches(row.content, CJK_CHAR_REGEX)
+    latin += countMatches(row.content, LATIN_LETTER_REGEX)
+  }
+
+  const total = cjk + latin
+  if (total === 0) return 'en'
+  return cjk / total >= CJK_RATIO_THRESHOLD ? 'zh' : 'en'
+}
+
+/** Resolve `auto` once, at the entry point, so Whisper never sees it. */
+export function resolveTranscriptionLanguage(
+  db: DatabaseAdapter,
+  language: TranscriptionLanguage
+): ResolvedTranscriptionLanguage {
+  return language === 'auto' ? detectSessionLanguage(db) : language
+}
+
+function countMatches(text: string, regex: RegExp): number {
+  regex.lastIndex = 0
+  let count = 0
+  while (regex.exec(text) !== null) count++
+  return count
+}
diff --git a/packages/node-runtime/src/index.ts b/packages/node-runtime/src/index.ts
@@ -499,8 +499,11 @@ export {
   planSessionTranscription,
   registerAudioDecoder,
   resolveTranscriptionModelCacheDir,
+  transcribeAttachmentPcm,
+  TranscribeAttachmentPcmError,
   transcribeSessionAttachments,
   DEFAULT_TRANSCRIPTION_PROFILE_ID,
+  MAX_TRANSCRIPTION_PCM_SAMPLES,
   TRANSCRIPTION_MODEL_CACHE_DIR_ENV,
   TRANSCRIPTION_PROFILE_IDS,
   TRANSCRIPTION_PROFILES,
@@ -509,6 +512,9 @@ export {
 export type {
   AudioDecoder,
   CreateTranscriberOptions,
+  ResolvedTranscriptionLanguage,
+  TranscribeAttachmentPcmErrorCode,
+  TranscribeAttachmentPcmResult,
   Transcriber,
   TranscriptionCandidate,
   TranscriptionLanguage,