Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
56de090
docs(contributing): build chatlab-mcp before running pnpm test
Zycheng114514 Sep 8, 2026
f750b4d
feat(electron): add UI scale setting
Zycheng114514 Sep 8, 2026
eb2488f
feat(tools): localize tool descriptions by locale for every runtime
Zycheng114514 Sep 8, 2026
a0065b1
feat(search): add message search benchmark script
Zycheng114514 Sep 8, 2026
e366771
feat(search): differentiate any/all queries in the search benchmark
Zycheng114514 Sep 8, 2026
6bae810
feat(search): add FTS5 trigram index for message content
Zycheng114514 Sep 8, 2026
919b033
feat(search): route keyword search through the FTS5 index
Zycheng114514 Sep 8, 2026
bb864f4
feat(search): build the message search index on Node import and migra…
Zycheng114514 Sep 8, 2026
a662cfa
feat(search): create the message search index in the browser runtime
Zycheng114514 Sep 8, 2026
c3d399c
feat(search): rank AI search hits by relevance and split deep search off
Zycheng114514 Sep 8, 2026
22b2ddd
test(search): cover the v11 migration in the desktop migration list
Zycheng114514 Sep 8, 2026
a944206
test(search): assert the import benchmark backfills the search index
Zycheng114514 Sep 8, 2026
532cce9
docs(contributing): note that chatlab-mcp must be rebuilt after sourc…
Zycheng114514 Sep 8, 2026
a5fe5cf
fix(electron): reapply the current UI scale after renderer reloads
Zycheng114514 Sep 8, 2026
09aa99c
test(import): reproduce cross-source duplicates when appending to a s…
Zycheng114514 Sep 8, 2026
e5d9867
feat(import): carry a chosen import target through the backends
Zycheng114514 Sep 8, 2026
ba0a61f
feat(import): let users choose the import target session
Zycheng114514 Sep 8, 2026
6d6b09a
fix(import): dedupe cross-source messages when appending to a session
Zycheng114514 Sep 8, 2026
65b4b96
docs(standard): define the attachments field of the ChatLab format
Zycheng114514 Sep 8, 2026
d339ec7
feat(semantic-index): reuse existing vectors for identical embedding …
Zycheng114514 Sep 8, 2026
ba7757d
feat(semantic-index): carry the semantic index over to a merged session
Zycheng114514 Sep 8, 2026
531747c
feat(merge): hand the merged session to the semantic index
Zycheng114514 Sep 8, 2026
3ddbe0e
feat(core): add message_attachment table and meta.source_dir
Zycheng114514 Sep 8, 2026
2f04ba0
feat(parser): emit attachments for ChatLab, Telegram, Discord and Ins…
Zycheng114514 Sep 8, 2026
4378155
feat(records): show message attachments
Zycheng114514 Sep 8, 2026
740b92c
chore(parser-native): regenerate the wasm bindings with attachment re…
Zycheng114514 Sep 8, 2026
581ef1e
docs(standard): point the pull protocol's attachments at the format spec
Zycheng114514 Sep 8, 2026
67aeb8b
merge: integrate docs/test-prereq-mcp-build into round-1
Zycheng114514 Sep 8, 2026
22f8e36
merge: integrate feat/ui-scale into round-1
Zycheng114514 Sep 8, 2026
796f538
merge: integrate feat/tool-descriptions-i18n into round-1
Zycheng114514 Sep 8, 2026
4722bc1
merge: integrate feat/message-fts5 into round-1
Zycheng114514 Sep 8, 2026
1649244
merge: integrate feat/semantic-index-carryover into round-1
Zycheng114514 Sep 8, 2026
528df75
merge: integrate feat/import-target-choice into round-1
Zycheng114514 Sep 8, 2026
34675fe
merge: integrate feat/message-attachments into round-1
Zycheng114514 Sep 8, 2026
bb36af2
test(migrations): cover the v11 and v12 upgrade path on one v10 database
Zycheng114514 Sep 8, 2026
486037a
feat(parser): add Telegram parser benchmark script
Zycheng114514 Sep 8, 2026
d3cde6b
feat(parser-native): add Telegram kernel
Zycheng114514 Sep 8, 2026
1552ba9
feat(web): parse Telegram exports with the Rust WASM kernel
Zycheng114514 Sep 8, 2026
0e384a6
feat(parser): add Discord parser benchmark script
Zycheng114514 Sep 8, 2026
2ac7bf1
feat(parser-native): add Discord kernel
Zycheng114514 Sep 8, 2026
4235609
feat(parser-native): scan Telegram chats with the Rust kernel
Zycheng114514 Sep 8, 2026
402a02e
merge: integrate feat/native-telegram-discord into round-1
Zycheng114514 Sep 8, 2026
5873957
feat(transcription): add local whisper transcription service
Zycheng114514 Sep 8, 2026
0bc4533
feat(cli): add transcribe command
Zycheng114514 Sep 8, 2026
91694c5
feat(transcription): add transcription benchmark script
Zycheng114514 Sep 8, 2026
4a6de9c
feat(transcription): detect the session language and transcribe PCM b…
Zycheng114514 Sep 8, 2026
c777ec9
refactor(node-runtime): share the worker thread transport
Zycheng114514 Sep 8, 2026
9bcf369
feat(transcription): run whisper in a worker thread
Zycheng114514 Sep 9, 2026
7789595
feat(electron): add transcription IPC
Zycheng114514 Sep 9, 2026
b304370
feat(cli-web): add transcription routes
Zycheng114514 Sep 9, 2026
7dabd77
feat(transcription): add the renderer audio decoder and platform service
Zycheng114514 Sep 9, 2026
09ed636
feat(records): transcribe voice attachments from the session page
Zycheng114514 Sep 9, 2026
6876ae0
feat(settings): add transcription model and language settings
Zycheng114514 Sep 9, 2026
8ceaecc
fix(records): refresh the session however the transcription modal is …
Zycheng114514 Sep 9, 2026
85f73c8
merge: integrate feat/voice-transcription into round-1
Zycheng114514 Sep 9, 2026
a33137b
feat(transcription): normalize Chinese transcripts to the session's s…
Zycheng114514 Sep 9, 2026
eb9d079
feat(settings): choose the Chinese script for transcripts
Zycheng114514 Sep 9, 2026
51dad5f
merge: integrate feat/transcription-chinese-script into round-1
Zycheng114514 Sep 9, 2026
a91ecb7
fix(records): show the voice transcription tool in the header tools menu
Zycheng114514 Sep 9, 2026
8e59cb1
merge: integrate fix/transcription-header-entry into round-1
Zycheng114514 Sep 9, 2026
5c26933
refactor(records): decode voice attachments directly at 16 kHz
Zycheng114514 Sep 9, 2026
d121ed1
merge: integrate fix/transcription-decode-at-16k into round-1
Zycheng114514 Sep 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat(import): carry a chosen import target through the backends
Add AutoImportOptions.forceCreate, which plans a separate session with
create reason 'user-choice' regardless of what automatic matching would
find. Expose the existing analyzeAutoImportFile preview through the
desktop worker (chat:analyzeAutoImport) and a CLI Web
/_web/import/analyze-target route so the renderer can preselect the
target automatic matching would pick.

Transport: chat:importWithOptions accepts a `target` option and maps it
to explicitSessionId / forceCreate; /_web/import accepts targetMode and
targetSessionId; /_web/import/batch accepts only auto and new, since
appending several files to one chosen session has no defined meaning;
clb import gains --new-session, mutually exclusive with --session-id.

Refs #376

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
  • Loading branch information
Zycheng114514 and claude committed Sep 8, 2026
commit e5d9867cdbdf43fad65f3fc4e60270107aef3513
133 changes: 129 additions & 4 deletions apps/cli/src/http/routes/web/import.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import multipart from '@fastify/multipart'

import { registerImportRoutes } from './import'

function multipartPayload(): { payload: Buffer; contentType: string } {
function multipartPayload(extraFields: Record<string, string> = {}): { payload: Buffer; contentType: string } {
const boundary = '----chatlab-auto-import-route-test'
const field = (name: string, value: string) =>
[`--${boundary}`, `Content-Disposition: form-data; name="${name}"`, '', value].join('\r\n')
Expand All @@ -24,6 +24,7 @@ function multipartPayload(): { payload: Buffer; contentType: string } {
field('formatId', 'telegram-json'),
field('chatIndex', '2'),
field('sessionGapThreshold', '7200'),
...Object.entries(extraFields).map(([name, value]) => field(name, value)),
file,
`--${boundary}--`,
'',
Expand Down Expand Up @@ -61,9 +62,14 @@ function directoryMultipartPayload(): { payload: Buffer; contentType: string } {
}
}

function batchMultipartPayload(): { payload: Buffer; contentType: string } {
function batchMultipartPayload(targetMode?: string): { payload: Buffer; contentType: string } {
const boundary = '----chatlab-batch-import-route-test'
const field = [`--${boundary}`, 'Content-Disposition: form-data; name="sessionGapThreshold"', '', '7200'].join('\r\n')
const fields = [
[`--${boundary}`, 'Content-Disposition: form-data; name="sessionGapThreshold"', '', '7200'].join('\r\n'),
...(targetMode === undefined
? []
: [[`--${boundary}`, 'Content-Disposition: form-data; name="targetMode"', '', targetMode].join('\r\n')]),
]
const files = ['first.json', 'second.json'].map((filename, index) =>
[
`--${boundary}`,
Expand All @@ -74,7 +80,7 @@ function batchMultipartPayload(): { payload: Buffer; contentType: string } {
].join('\r\n')
)
return {
payload: Buffer.from([field, ...files, `--${boundary}--`, ''].join('\r\n')),
payload: Buffer.from([...fields, ...files, `--${boundary}--`, ''].join('\r\n')),
contentType: `multipart/form-data; boundary=${boundary}`,
}
}
Expand Down Expand Up @@ -371,4 +377,123 @@ describe('CLI Web automatic import route', () => {
await app.close()
}
})

it('maps the requested import target onto the auto importer options', async () => {
const cases: Array<{
fields: Record<string, string>
expected: { sessionId?: string; forceCreate?: boolean }
}> = [
{ fields: {}, expected: { sessionId: undefined, forceCreate: undefined } },
{ fields: { targetMode: 'auto' }, expected: { sessionId: undefined, forceCreate: undefined } },
{ fields: { targetMode: 'new' }, expected: { sessionId: undefined, forceCreate: true } },
{
fields: { targetMode: 'session', targetSessionId: 'chat_42' },
expected: { sessionId: 'chat_42', forceCreate: undefined },
},
]

for (const { fields, expected } of cases) {
const app = Fastify()
const calls: Array<{ sessionId?: string; forceCreate?: boolean }> = []
try {
await app.register(multipart)
registerImportRoutes(
app,
{} as any,
{
runAutoImport: async (_filePath: string, options: Record<string, unknown>) => {
calls.push({
sessionId: options.sessionId as string | undefined,
forceCreate: options.forceCreate as boolean | undefined,
})
return { success: true, sessionId: 'chat_42', importMode: 'incremental' }
},
} as any
)

const body = multipartPayload(fields)
const response = await app.inject({
method: 'POST',
url: '/_web/import',
headers: { 'content-type': body.contentType },
payload: body.payload,
})

assert.equal(response.statusCode, 200)
assert.deepEqual(calls, [expected])
} finally {
await app.close()
}
}
})

it('rejects an import target that names a session but no session id', async () => {
const app = Fastify()
try {
await app.register(multipart)
registerImportRoutes(
app,
{} as any,
{
runAutoImport: async () => {
throw new Error('import must not start')
},
} as any
)

const body = multipartPayload({ targetMode: 'session' })
const response = await app.inject({
method: 'POST',
url: '/_web/import',
headers: { 'content-type': body.contentType },
payload: body.payload,
})

assert.equal(response.statusCode, 400)
assert.equal(response.json().error, 'targetSessionId is required when targetMode is "session"')
} finally {
await app.close()
}
})

it('lets a batch ask for separate sessions but refuses one shared session target', async () => {
const app = Fastify()
const forceCreateFlags: Array<boolean | undefined> = []
try {
await app.register(multipart)
registerImportRoutes(
app,
{} as any,
{
runAutoImportBatch: async (items: any[]) => {
for (const item of items) forceCreateFlags.push(item.forceCreate)
return items.map((item) => ({ id: item.id, status: 'success', result: { success: true } }))
},
} as any
)

const newBody = batchMultipartPayload('new')
const newResponse = await app.inject({
method: 'POST',
url: '/_web/import/batch',
headers: { 'content-type': newBody.contentType },
payload: newBody.payload,
})

const sessionBody = batchMultipartPayload('session')
const sessionResponse = await app.inject({
method: 'POST',
url: '/_web/import/batch',
headers: { 'content-type': sessionBody.contentType },
payload: sessionBody.payload,
})

assert.equal(newResponse.statusCode, 200)
assert.deepEqual(forceCreateFlags, [true, true])
assert.equal(sessionResponse.statusCode, 400)
assert.equal(sessionResponse.json().error, 'targetMode "session" is not supported for batch imports')
} finally {
await app.close()
}
})
})
82 changes: 82 additions & 0 deletions apps/cli/src/http/routes/web/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
streamImport,
incrementalImport,
analyzeIncrementalImport,
analyzeAutoImport,
analyzeNewImport,
detectFormat,
detectAllFormats,
Expand All @@ -27,6 +28,7 @@ import {
findEntryFileInDirectory,
} from '../../../import'
import type {
AutoImportAnalysisResult,
AutoImportBatchOptions,
AutoImportBatchRequest,
AutoImportResult,
Expand All @@ -39,6 +41,7 @@ const ARCHIVE_UPLOAD_LIMIT = 50 * 1024 * 1024 * 1024
interface ImportRouteOptions {
sourceManager?: ArchiveImportSourceManager
runAutoImport?: (filePath: string, options: StreamImportOptions) => Promise<AutoImportResult>
runAnalyzeAutoImport?: (filePath: string, options: StreamImportOptions) => Promise<AutoImportAnalysisResult>
runAutoImportBatch?: (
items: AutoImportBatchRequest[],
options: AutoImportBatchOptions
Expand All @@ -57,6 +60,31 @@ function parseOptionalInteger(value: unknown): number | undefined {
return Number.isInteger(parsed) ? parsed : undefined
}

interface ImportTargetFields {
sessionId?: string
forceCreate?: boolean
}

/**
* Map the requested import target onto the auto importer's own options.
* `allowSession` is false for batch endpoints, where appending several files to one
* chosen session has no well-defined meaning.
*/
function resolveImportTarget(
mode: unknown,
sessionId: unknown,
allowSession: boolean
): { target: ImportTargetFields } | { error: string } {
if (mode === undefined || mode === '' || mode === 'auto') return { target: {} }
if (mode === 'new') return { target: { forceCreate: true } }
if (mode !== 'session') return { error: `Unsupported targetMode: ${String(mode)}` }
if (!allowSession) return { error: 'targetMode "session" is not supported for batch imports' }
if (typeof sessionId !== 'string' || sessionId === '') {
return { error: 'targetSessionId is required when targetMode is "session"' }
}
return { target: { sessionId } }
}

function cleanupTemp(...paths: string[]) {
for (const p of paths) {
try {
Expand All @@ -80,6 +108,7 @@ export function registerImportRoutes(
const activeBatchControllers = new Map<string, AbortController>()
const sourceManager = options.sourceManager ?? new ArchiveImportSourceManager()
const runAutoImport = options.runAutoImport ?? autoImport.bind(null, dbManager)
const runAnalyzeAutoImport = options.runAnalyzeAutoImport ?? analyzeAutoImport.bind(null, dbManager)
const runAutoImportBatch = options.runAutoImportBatch ?? autoImportBatch.bind(null, dbManager)
const runPreparedImport =
options.runPreparedImport ??
Expand Down Expand Up @@ -232,6 +261,15 @@ export function registerImportRoutes(
const chatIndexStr = (data.fields?.chatIndex as any)?.value as string | undefined
const chatIndex = chatIndexStr !== undefined ? parseInt(chatIndexStr, 10) : undefined
const sessionGapThreshold = parseOptionalInteger((data.fields?.sessionGapThreshold as any)?.value)
const target = resolveImportTarget(
(data.fields?.targetMode as any)?.value,
(data.fields?.targetSessionId as any)?.value,
true
)
if ('error' in target) {
cleanupTemp(tmpPath, tmpDir)
return reply.code(400).send({ error: target.error })
}

reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
Expand All @@ -249,6 +287,8 @@ export function registerImportRoutes(
formatId,
chatIndex,
sessionGapThreshold,
sessionId: target.target.sessionId,
forceCreate: target.target.forceCreate,
nativeBinding,
onProgress: (p) => sendEvent('progress', p),
})
Expand Down Expand Up @@ -279,6 +319,7 @@ export function registerImportRoutes(
const tmpDir = createChatLabTempDir('imports', 'batch-')
const items: AutoImportBatchRequest[] = []
let sessionGapThreshold: number | undefined
let targetMode: unknown
let completed = false
const controller = new AbortController()
activeBatchControllers.set(batchId, controller)
Expand All @@ -289,6 +330,10 @@ export function registerImportRoutes(
sessionGapThreshold = parseOptionalInteger(part.value)
continue
}
if (part.type === 'field' && part.fieldname === 'targetMode') {
targetMode = part.value
continue
}
if (part.type !== 'file') continue
const index = items.length
const safeName = path.basename(part.filename || `upload-${index}`)
Expand All @@ -299,6 +344,12 @@ export function registerImportRoutes(

if (items.length === 0) return reply.code(400).send({ error: 'No files uploaded' })

const target = resolveImportTarget(targetMode, undefined, false)
if ('error' in target) return reply.code(400).send({ error: target.error })
if (target.target.forceCreate) {
for (const item of items) item.forceCreate = true
}

reply.raw.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Expand Down Expand Up @@ -422,6 +473,37 @@ export function registerImportRoutes(
}
})

// Preview which target automatic matching would pick, so the import dialog can preselect it.
server.post('/_web/import/analyze-target', async (request, reply) => {
const data = await (request as any).file()
if (!data) return reply.code(400).send({ error: 'No file uploaded' })

const tmpDir = createChatLabTempDir('imports', 'analyze-target-')
const tmpPath = path.join(tmpDir, data.filename || 'upload')

const chunks: Buffer[] = []
for await (const chunk of data.file) {
chunks.push(chunk)
}
fs.writeFileSync(tmpPath, Buffer.concat(chunks))

const formatId = (data.fields?.formatId as any)?.value as string | undefined
const chatIndexStr = (data.fields?.chatIndex as any)?.value as string | undefined
const chatIndex = chatIndexStr !== undefined ? parseInt(chatIndexStr, 10) : undefined

try {
return await runAnalyzeAutoImport(tmpPath, {
formatId,
chatIndex,
nativeBinding: resolveNativeBinding(),
})
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) })
} finally {
cleanupTemp(tmpPath, tmpDir)
}
})

// ==================== Incremental Import ====================

server.post<{ Params: { id: string } }>('/_web/sessions/:id/import/incremental', async (request, reply) => {
Expand Down
37 changes: 36 additions & 1 deletion apps/cli/src/import/command.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { buildImportCommandEnvelope, withJsonDiagnosticsOnStderr } from './command'
import { Command } from 'commander'
import { buildImportCommandEnvelope, registerImportCommand, withJsonDiagnosticsOnStderr } from './command'

test('buildImportCommandEnvelope exposes a stable dry-run success contract', () => {
assert.deepEqual(
Expand Down Expand Up @@ -66,3 +67,37 @@ test('withJsonDiagnosticsOnStderr keeps runtime diagnostics out of machine-reada
assert.deepEqual(stdout, [])
assert.deepEqual(stderr, ['[Parser V2] Using parser: ChatLab JSON'])
})

test('import rejects --session-id together with --new-session before touching the data directory', async () => {
const program = new Command()
program.exitOverride()
registerImportCommand(program)

const stdout: string[] = []
const originalWrite = process.stdout.write.bind(process.stdout)
const originalExitCode = process.exitCode
process.stdout.write = ((chunk: string) => {
stdout.push(String(chunk))
return true
}) as typeof process.stdout.write

try {
await program.parseAsync(['import', 'missing-file.json', '--json', '--session-id', 'target', '--new-session'], {
from: 'user',
})
} finally {
process.stdout.write = originalWrite
process.exitCode = originalExitCode
}

assert.deepEqual(JSON.parse(stdout.join('')), {
ok: false,
command: 'import',
error: {
code: 'CONFLICTING_TARGET_OPTIONS',
message: 'Use either --session-id or --new-session, not both',
hint: '--session-id appends to an existing session; --new-session always creates a separate one.',
},
meta: { dryRun: false, apiVersion: 1 },
})
})
Loading