diff --git a/docs/LOCAL_MODELS.md b/docs/LOCAL_MODELS.md index 453b340..bac07e9 100644 --- a/docs/LOCAL_MODELS.md +++ b/docs/LOCAL_MODELS.md @@ -351,6 +351,30 @@ Creative quality and native-English listening acceptance remain for human review ## Follow-up reuse from the existing story scripts +### Fixed-reference narration in this fork + +For the workstation's authenticated remote MCP gateway, `bdesign_asset_upload` +accepts canonical base64 media in 512 KiB decoded chunks. An upload UUID is bound +to the project, filename, total byte length and SHA-256; identical chunks can be +retried, conflicting bytes are rejected. The file limit is 64 MiB. Final import +uses the existing project Asset service and ownership checks. This transports +only explicitly supplied bytes and does not expose the blocked host-path import +tool or change gateway/HTTP authentication. Keep upload receipts for recovery. + +`bdesign_audio_generate` accepts `referenceAssetId` plus its verbatim `referenceText`, +and optional `maxNewTokens` (32–1024), `temperature` (0–2), and `topP` (0–1, exclusive zero). +Text remains limited to 200 characters per take. Omit both reference fields to +retain unconditioned generation. UI/API asset-first requests use an `audio_track` +reference and `reference_text`, `max_new_tokens`, `temperature`, `top_p` parameters. +The shared preflight enforces project ownership, local audio media and one +reference. The worker validates a decodable 0–30 second reference and records its +Asset ID, SHA-256, exact transcript, parameters and input fingerprint for each take. +It maps Windows drive paths to the existing WSL Drama service. No client-supplied +filesystem path, remote provider fallback or separate project state is introduced. +Interrupted/ambiguous TTS requests are not automatically repeated. Restart web/MCP +after installing this code. Actual voice quality and listening acceptance remain +distinct from automated request/provenance checks. + The story-production handoff was reviewed separately. The pinned H3 output keeps its original audio and native duration; adding narration does not replace that source Asset. Do not import the old story scripts as libraries: several execute diff --git a/scripts/local-worker.py b/scripts/local-worker.py index c56b75a..faf72d4 100644 --- a/scripts/local-worker.py +++ b/scripts/local-worker.py @@ -35,7 +35,7 @@ def main(): from local_h3_duration import execute_h3_with_duration, verify_h3_output from local_ref2va import execute_ref2va from local_flux3 import ensure_flux3, execute_flux3, release_flux3 - from tools.audio.drama3_local_tts import Drama3LocalTTS + from local_narration import execute_narration settings = json.loads(Path(args.settings).read_text(encoding='utf8')) if args.settings else {} flux_config = settings.get('flux3') @@ -106,7 +106,7 @@ def notify(message): if _ready(bootstrap()['comfyui_url'] + '/system_stats'): release_comfy() ensure_drama(notify) - result = Drama3LocalTTS().execute({'text': inputs['prompt'], 'output_path': str(output)}) + result = execute_narration(inputs, output) if not result.success: raise RuntimeError(result.error) provenance = result.data diff --git a/scripts/local_narration.py b/scripts/local_narration.py new file mode 100644 index 0000000..0fdb545 --- /dev/null +++ b/scripts/local_narration.py @@ -0,0 +1,93 @@ +"""Local Drama adapter called only by the normal BeatDesign job worker. + +References are typed project Assets resolved by shared server preflight, never +client-supplied paths. Reuses the runtime's existing service and GPU lease. +""" +import hashlib +import json +from pathlib import Path, PureWindowsPath +import time +from urllib.parse import urlparse + + +def server_audio_path(path): + """The configured Drama service runs in WSL on the same workstation.""" + win = PureWindowsPath(str(path)) + if win.drive: + if len(win.drive) != 2 or win.drive[1] != ':': + raise ValueError('Drama reference must be on a WSL-mounted local drive, not UNC') + return '/mnt/' + win.drive[0].lower() + '/' + '/'.join(win.parts[1:]) + return str(Path(path).resolve()) + + +def describe_narration(inputs, probe): + text = inputs.get('prompt') + if not isinstance(text, str) or not text.strip() or len(text) > 200: + raise ValueError('Narration requires 1-200 characters') + payload = {'input': text, 'response_format': 'wav', 'stream': False, + 'max_new_tokens': inputs.get('max_new_tokens', 512), + 'temperature': inputs.get('temperature', 0.8), 'top_p': inputs.get('top_p', 0.9)} + refs = inputs.get('_localReferences', []) + ref_text = inputs.get('reference_text') + if bool(refs) != bool(ref_text) or len(refs) > 1: + raise ValueError('One reference audio Asset and exact reference text must be supplied together') + references = [] + if refs: + ref = refs[0] + if ref.get('type') != 'audio' or not ref.get('assetId'): + raise ValueError('Narration reference must be a resolved audio Asset') + path = Path(ref['path']).resolve(strict=True) + media = probe(path) + if not any(s['codec_type'] == 'audio' for s in media['streams']): + raise ValueError('Reference has no decodable audio') + duration = float(media['format']['duration']) + if not 0 < duration <= 30: + raise ValueError('Reference voice must be greater than zero and at most 30 seconds') + payload['references'] = [{'audio_path': server_audio_path(path), 'text': ref_text}] + references.append({'assetId': ref['assetId'], 'path': str(path), 'text': ref_text, + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'duration_seconds': duration}) + fingerprint = hashlib.sha256(json.dumps({'request': payload, 'references': references}, sort_keys=True).encode()).hexdigest() + return payload, references, fingerprint + + +def execute_narration(inputs, output): + import requests + from lib.local_runtime import bootstrap, probe_media, write_json + from tools.base_tool import ToolResult + started = time.monotonic() + try: + cfg = bootstrap() + base = cfg['drama3_url'].rstrip('/') + if urlparse(base).hostname not in {'localhost', '127.0.0.1', '::1'}: + raise ValueError('The configured Drama worker must be local') + output = Path(output).resolve() + if output.exists() or output.with_suffix('.request.json').exists(): + raise FileExistsError('Ambiguous/previous narration attempt exists; inspect before creating a new take') + payload, references, fingerprint = describe_narration(inputs, probe_media) + models = requests.get(base + '/v1/models', timeout=10) + models.raise_for_status() + model_info = models.json() + if not any('drama-3-int4' in m.get('id', '') for m in model_info.get('data', [])): + raise RuntimeError('Configured service does not report Drama 3 INT4') + output.parent.mkdir(parents=True, exist_ok=True) + write_json(output.with_suffix('.request.json'), {'request': payload, 'references': references, 'input_fingerprint': fingerprint}) + response = requests.post(base + '/v1/audio/speech', json=payload, timeout=(10, 900)) + if not response.ok: + raise RuntimeError(f'Drama HTTP {response.status_code}: {response.text[:1000]}') + if response.content[:4] != b'RIFF': + raise RuntimeError('Drama did not return WAV audio') + output.write_bytes(response.content) + media = probe_media(output) + audio = next(s for s in media['streams'] if s['codec_type'] == 'audio') + provenance = {'provider': 'drama3_local', 'model': cfg['drama3_model'], 'output': str(output), + 'text': payload['input'], 'audio_duration_seconds': float(media['format']['duration']), + 'sample_rate': int(audio['sample_rate']), 'channels': audio['channels'], + 'sha256': hashlib.sha256(response.content).hexdigest(), 'references': references, + 'input_fingerprint': fingerprint, 'voice_mode': 'fixed_reference' if references else 'unconditional', + 'parameters': {k: v for k, v in payload.items() if k not in ('references', 'input')}, + 'endpoint_models': model_info, 'human_listening_accepted': False} + write_json(output.with_suffix('.provenance.json'), provenance) + return ToolResult(success=True, data=provenance, artifacts=[str(output)], + model=cfg['drama3_model'], duration_seconds=time.monotonic()-started) + except Exception as exc: + return ToolResult(success=False, error=str(exc), duration_seconds=time.monotonic()-started) diff --git a/scripts/test_local_narration.py b/scripts/test_local_narration.py new file mode 100644 index 0000000..c98b376 --- /dev/null +++ b/scripts/test_local_narration.py @@ -0,0 +1,40 @@ +from pathlib import Path +import tempfile +import unittest +from local_narration import describe_narration, server_audio_path + + +class NarrationTests(unittest.TestCase): + def probe(self, path): + return {'streams': [{'codec_type': 'audio'}], 'format': {'duration': '12'}} + + def test_windows_reference_mount(self): + self.assertEqual(server_audio_path(r'D:\Project with spaces\voice.wav'), '/mnt/d/Project with spaces/voice.wav') + with self.assertRaises(ValueError): + server_audio_path(r'\\remote\share\voice.wav') + + def test_reference_hash_transcript_and_redo_identity(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / 'voice.wav'; path.write_bytes(b'RIFFfixtureWAVE') + inputs = {'prompt': 'Come back, cat!', 'reference_text': 'A lovely day.', + '_localReferences': [{'assetId': 'voice', 'path': str(path), 'type': 'audio'}]} + payload, refs, fingerprint = describe_narration(inputs, self.probe) + self.assertEqual(payload['references'][0]['text'], inputs['reference_text']) + self.assertEqual(refs[0]['assetId'], 'voice') + self.assertEqual(len(refs[0]['sha256']), 64) + self.assertEqual(describe_narration(inputs, self.probe)[2], fingerprint) + next_payload, next_refs, _ = describe_narration({**inputs, 'prompt': 'Stay here!'}, self.probe) + self.assertEqual(next_refs, refs) + self.assertEqual(next_payload['references'], payload['references']) + path.write_bytes(b'changed');self.assertNotEqual(describe_narration(inputs, self.probe)[2], fingerprint) + + def test_reference_limits_and_pairing(self): + with self.assertRaises(ValueError):describe_narration({'prompt':'Hello', 'reference_text':'Ref'}, self.probe) + with self.assertRaises(ValueError):describe_narration({'prompt':'x'*201}, self.probe) + with tempfile.TemporaryDirectory() as root: + path = Path(root) / 'voice.wav';path.write_bytes(b'RIFF') + inputs={'prompt':'Hi', 'reference_text':'Ref','_localReferences':[{'assetId':'voice','type':'audio','path':str(path)}]} + with self.assertRaises(ValueError):describe_narration(inputs, lambda p:{'streams':[{'codec_type':'audio'}],'format':{'duration':'31'}}) + + +if __name__ == '__main__':unittest.main() diff --git a/src/core/effects/registered-effects.ts b/src/core/effects/registered-effects.ts index c83a4fc..45d9f5e 100644 --- a/src/core/effects/registered-effects.ts +++ b/src/core/effects/registered-effects.ts @@ -50,7 +50,18 @@ const buildVideoAnalysisEffect = ( }; }; -const buildLocalAudioEffect = (binding: GenerationProviderModelBinding, providerId = getActiveGenerationProviderId()): RegisteredEffect => ({ id: binding.effectId, name: LOCAL_AUDIO_MODEL.name, type: 4, model: binding.modelId, version: null, linkName: binding.modelId, description: 'Local short narration', platform: providerId, api: null, provider: providerId, inputSchema: { prompt: { type: 'string', required: true, maxLength: 200 } } }); +const buildLocalAudioEffect = (binding: GenerationProviderModelBinding, providerId = getActiveGenerationProviderId()): RegisteredEffect => ({ + id: binding.effectId, name: LOCAL_AUDIO_MODEL.name, type: 4, model: binding.modelId, + version: null, linkName: binding.modelId, description: 'Local narration with optional fixed reference voice', + platform: providerId, api: null, provider: providerId, + inputSchema: { + prompt: { type: 'string', required: true, maxLength: 200 }, + reference_text: { type: 'string', required: false, maxLength: 4000 }, + max_new_tokens: { type: 'number', required: false, minimum: 32, maximum: 1024 }, + temperature: { type: 'number', required: false, minimum: 0, maximum: 2 }, + top_p: { type: 'number', required: false, exclusiveMinimum: 0, maximum: 1 }, + }, +}); const optionalAnyField = { type: 'any', required: false } as const; diff --git a/src/core/generation-providers/local-drama3.ts b/src/core/generation-providers/local-drama3.ts new file mode 100644 index 0000000..0401617 --- /dev/null +++ b/src/core/generation-providers/local-drama3.ts @@ -0,0 +1,23 @@ +import { z } from 'zod'; + +export const LOCAL_NARRATION_MAX_TEXT = 200; +export const localNarrationParameters = z.object({ + reference_text: z.string().trim().min(1).max(4000).optional(), + max_new_tokens: z.number().int().min(32).max(1024).optional(), + temperature: z.number().min(0).max(2).optional(), + top_p: z.number().gt(0).max(1).optional(), +}).strict(); + +export function validateLocalNarration(input: Record) { + if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > LOCAL_NARRATION_MAX_TEXT) + throw new Error('Drama 3 narration requires text up to 200 characters.'); + if (input.image_urls || input.video_urls) throw new Error('Drama 3 accepts audio references only.'); + const { prompt: _prompt, audio_urls, _localReferences: _refs, _localReferencePaths: _paths, ...parameters } = input; + const parsed = localNarrationParameters.safeParse(parameters); + if (!parsed.success) throw new Error(`Unsupported Drama 3 parameter: ${parsed.error.message}`); + if (audio_urls !== undefined && (!Array.isArray(audio_urls) || audio_urls.length !== 1 || + audio_urls.some(url => typeof url !== 'string' || !url.trim()))) + throw new Error('Drama 3 accepts exactly one reference audio when supplied.'); + if (Boolean(audio_urls) !== Boolean(parsed.data.reference_text)) + throw new Error('Reference audio and its exact reference_text must be supplied together.'); +} diff --git a/src/core/generation-providers/local-provider.ts b/src/core/generation-providers/local-provider.ts index 086a0dc..679f150 100644 --- a/src/core/generation-providers/local-provider.ts +++ b/src/core/generation-providers/local-provider.ts @@ -5,6 +5,7 @@ import { createServerOnlyFn } from '@tanstack/react-start'; import { requireLocalH3Timing } from './local-h3-duration'; import referenceLimits from './local-h3-ref2va.json'; import flux3 from './local-flux3.json'; +import { validateLocalNarration } from './local-drama3'; const submit = createServerOnlyFn(async (model: string, input: unknown) => (await import('./local-runtime.server')).submitLocalJob(model, input)); @@ -41,6 +42,7 @@ export const localGenerationProvider: GenerationProviderDefinition = { }, validateInput: (effect, input) => { if (!LOCAL_MODELS.includes(effect.model)) return beatApiGenerationProvider.validateInput?.(effect, input); + if (effect.model === 'local-drama3') return validateLocalNarration(input); if (effect.model === 'local-h3' && (!Array.isArray(input.image_urls) || input.image_urls.length !== 1)) throw new Error('Local H3 requires exactly one first-frame image.'); if (effect.model === 'local-h3' && (input.video_urls || input.audio_urls || @@ -70,8 +72,6 @@ export const localGenerationProvider: GenerationProviderDefinition = { throw new Error(`Local FLUX 3 accepts at most ${flux3.maxReferenceImages} image references.`); const allowed = new Set(['prompt', '_localReferencePaths', '_localReferences', ...(isH3 ? ['image_urls', 'seed', 'aspect_ratio', 'wmDuration'] : []), ...(effect.model === 'local-h3-ref2va' ? ['video_urls', 'audio_urls'] : []), ...(effect.model === 'local-flux3-klein' ? ['image_urls', 'seed', 'aspect_ratio'] : [])]); if (Object.keys(input).some(key => !allowed.has(key))) throw new Error('Unsupported local model parameter.'); - if (effect.model === 'local-drama3' && (String(input.prompt).length > 200 || input.image_urls || input.audio_urls || input.video_urls)) - throw new Error('Drama 3 preview accepts text only, up to 200 characters.'); if (input.seed !== undefined && (!Number.isSafeInteger(input.seed) || Number(input.seed) < 0)) throw new Error('Seed must be a non-negative safe integer.'); }, diff --git a/src/core/generation-providers/model-catalog.ts b/src/core/generation-providers/model-catalog.ts index 21aa2c8..e1aef82 100644 --- a/src/core/generation-providers/model-catalog.ts +++ b/src/core/generation-providers/model-catalog.ts @@ -49,7 +49,11 @@ export function listGenerationModelDescriptors(): GenerationModelDescriptor[] { return provider.modelBindings.flatMap((binding) => { const effect = getRegisteredEffectById(binding.effectId); if (!effect) return []; - if (effect.type === 4) return [{ id: binding.modelId, name: effect.name, providerId: provider.id, kind: 'audio', available: true, parameterSchema: effect.inputSchema, defaultParameters: {}, referenceSchema: { image: { min: 0, max: 0 }, video: { min: 0, max: 0 }, audio: { min: 0, max: 0 } } }]; + if (effect.type === 4) return [{ id: binding.modelId, name: effect.name, providerId: provider.id, kind: 'audio', available: true, parameterSchema: effect.inputSchema, + defaultParameters: { max_new_tokens: 512, temperature: 0.8, top_p: 0.9 }, + referenceSchema: { image: { min: 0, max: 0 }, video: { min: 0, max: 0 }, audio: { min: 0, max: 1 }, + maxTotal: 1, audioDurationSeconds: { minExclusive: 0, max: 30 }, + requiresReferenceText: true, note: 'One optional project-owned audio Asset and its exact transcript reuse the voice. Omit both for an unconditioned voice.' } }]; if (binding.modelId === VIDEO_ANALYSIS_MODEL_ID) { return [ { diff --git a/src/core/generation-providers/submit.integration.test.ts b/src/core/generation-providers/submit.integration.test.ts index 695346b..6038d58 100644 --- a/src/core/generation-providers/submit.integration.test.ts +++ b/src/core/generation-providers/submit.integration.test.ts @@ -337,6 +337,29 @@ test('local narration enforces the short-text boundary and is discoverable as au ); }); +test('fixed narration voice uses a project-owned audio Asset and identical UI/MCP compilation', async () => { + const { importLocalProjectAsset } = await import('@/core/projects/import-local-asset'); + const project = await createProject({ name: 'Fixed voice' }); + const other = await createProject({ name: 'Other voice owner' }); + const audioPath = join(directory, 'fixed-voice.wav'); + await writeFile(audioPath, Buffer.from('RIFFfixtureWAVE')); + const voice = await importLocalProjectAsset({ projectId: project.id, filePath: audioPath }); + const foreign = await importLocalProjectAsset({ projectId: other.id, filePath: audioPath }); + const generation = { version: 2, projectId: project.id, modelId: 'local-drama3', mode: 'audio', prompt: 'Come back, cat!', + references: [{assetId: voice.id, role: 'audio_track'}], + parameters: {reference_text: 'A lovely day.', max_new_tokens: 512, _localReferences: [{path:'C:/forged.wav',type:'audio'}]} }; + assert.equal((await preflightAssetFirstGeneration(generation)).expectedUploadCount, 0); + await submitAssetFirstGeneration({generation,origin:'ui'}); + const ui = inputs.at(-1) as Record; + await submitAssetFirstGeneration({generation,origin:'mcp'}); + assert.deepEqual(inputs.at(-1),ui); + const refs = ui._localReferences as Array<{assetId:string;path:string;type:string}>; + assert.equal(refs[0].assetId,voice.id); assert.equal(refs[0].type,'audio'); + assert.ok(!refs[0].path.includes('forged')); + await assert.rejects(()=>preflightAssetFirstGeneration({...generation,references:[{assetId:foreign.id,role:'audio_track'}]}), /does not belong/); + await assert.rejects(()=>preflightAssetFirstGeneration({...generation,parameters:{}}), /together/); +}); + test('local Ref2VA delivers ordered typed project references identically from UI and MCP without upload', async () => { const { importLocalProjectAsset } = await import('@/core/projects/import-local-asset'); const { copyFile } = await import('node:fs/promises'); diff --git a/src/core/generation-providers/submit.ts b/src/core/generation-providers/submit.ts index bad829a..2a8c2a5 100644 --- a/src/core/generation-providers/submit.ts +++ b/src/core/generation-providers/submit.ts @@ -270,9 +270,11 @@ export async function preflightAssetFirstGeneration(source: unknown) { for (const [assetId, asset] of uniqueAssets) { if (!asset) throw new Error(`Asset ${assetId} does not belong to this project.`); - if (['local-h3-ref2va', 'local-flux3-klein'].includes(generation.modelId)) { + if (['local-h3-ref2va', 'local-flux3-klein', 'local-drama3'].includes(generation.modelId)) { if (generation.modelId === 'local-flux3-klein' && asset.type !== 'image') throw new Error('Local FLUX 3 accepts image references only.'); + if (generation.modelId === 'local-drama3' && asset.type !== 'audio') + throw new Error('Local Drama 3 accepts audio references only.'); if (asset.storageProvider !== 'local') throw new Error('Import local model references into local project storage first.'); const file = await stat(resolveLocalProjectAssetPath({ objectKey: asset.objectKey })); const candidate = { name: asset.filename ?? '', type: asset.mimeType ?? '', size: file.size }; @@ -381,7 +383,7 @@ export async function submitAssetFirstGeneration({ references.push({ assetId: asset.id, type: asset.type, path }); } Object.assign(input, { _localReferencePaths: paths }); - if (['local-h3-ref2va', 'local-flux3-klein'].includes(generation.modelId)) Object.assign(input, { _localReferences: references }); + if (['local-h3-ref2va', 'local-flux3-klein', 'local-drama3'].includes(generation.modelId)) Object.assign(input, { _localReferences: references }); } validateGenerationModelInput({ modelId: generation.modelId, input }); const result = await submitEffectGeneration({ diff --git a/src/mcp/asset-upload.test.ts b/src/mcp/asset-upload.test.ts new file mode 100644 index 0000000..4b33723 --- /dev/null +++ b/src/mcp/asset-upload.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, test } from 'node:test'; + +const directory = await mkdtemp(join(tmpdir(),'beatdesign-mcp-upload-')); +process.env.BEATDESIGN_DATA_DIR=directory; +execFileSync(process.execPath,['node_modules/drizzle-kit/bin.cjs','push','--force'],{env:process.env,stdio:'pipe'}); +const {createProject}=await import('@/core/projects/projects'); +const {assetUploadInputSchema,uploadProjectAssetChunk,UPLOAD_CHUNK_BYTES}=await import('./asset-upload'); +const {getProjectAssetById}=await import('@/core/workspace-lib/assets/user-assets'); +after(async()=>{const {getDb}=await import('@/core/workspace-lib/db-adapter');(await getDb()).$client.close();await rm(directory,{recursive:true,force:true,maxRetries:10,retryDelay:100});}); + +test('chunk upload resumes, checks digest and imports the same project Asset without accepting paths',async()=>{ + const project=await createProject({name:'Upload'}); + const bytes=Buffer.concat([await readFile('public/logo.png'),Buffer.alloc(UPLOAD_CHUNK_BYTES)]); + const count=Math.ceil(bytes.length/UPLOAD_CHUNK_BYTES); + const base={uploadId:randomUUID(),filename:'cover.png',sizeBytes:bytes.length,sha256:createHash('sha256').update(bytes).digest('hex')}; + const chunk=(index:number)=>({...base,partIndex:index,dataBase64:bytes.subarray(index*UPLOAD_CHUNK_BYTES,(index+1)*UPLOAD_CHUNK_BYTES).toString('base64')}); + const first=await uploadProjectAssetChunk(project.id,chunk(0)); + assert.equal(first.complete,false); + assert.deepEqual(await uploadProjectAssetChunk(project.id,chunk(0)),first); + await assert.rejects(()=>uploadProjectAssetChunk(project.id,{...chunk(0),filename:'different.png'}),/different file/); + let result:any; + for(let i=1;i{ + const project=await createProject({name:'Bad bytes'}); + const data=await readFile('public/logo.png'); + const input={uploadId:randomUUID(),filename:'logo.png',sizeBytes:data.length,sha256:'0'.repeat(64),partIndex:0,dataBase64:data.toString('base64')}; + await assert.rejects(()=>uploadProjectAssetChunk(project.id,input),/checksum/); + await assert.rejects(()=>uploadProjectAssetChunk(project.id,{...input,uploadId:randomUUID(),sizeBytes:data.length+1}),/size/); +}); diff --git a/src/mcp/asset-upload.ts b/src/mcp/asset-upload.ts new file mode 100644 index 0000000..ff451e6 --- /dev/null +++ b/src/mcp/asset-upload.ts @@ -0,0 +1,82 @@ +import { createHash } from 'node:crypto'; +import { mkdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { z } from 'zod'; +import { getBeatDesignDataRoot } from '@/config/data-root'; +import { getProject } from '@/core/projects/projects'; +import { importLocalProjectAsset, validateLocalAssetImportCandidate } from '@/core/projects/import-local-asset'; + +export const UPLOAD_CHUNK_BYTES = 512 * 1024; +export const assetUploadInputSchema = z.object({ + projectId: z.string().min(1).max(160).optional(), + uploadId: z.string().uuid(), + filename: z.string().min(1).max(180).regex(/^[^<>:"/\\|?*\x00-\x1f]+$/), + sizeBytes: z.number().int().min(1).max(64 * 1024 * 1024), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + partIndex: z.number().int().min(0).max(127), + dataBase64: z.string().min(4).max(4 * Math.ceil(UPLOAD_CHUNK_BYTES / 3)), +}).strict(); + +const pending = new Map>(); +export async function uploadProjectAssetChunk(projectId: string, source: z.infer) { + const input = assetUploadInputSchema.parse(source); + const key = `${projectId}:${input.uploadId}`; + const previous = pending.get(key) ?? Promise.resolve(); + const operation = previous.catch(() => {}).then(() => saveChunk(projectId, input)); + pending.set(key, operation); + try { return await operation; } finally { if (pending.get(key) === operation) pending.delete(key); } +} + +async function saveChunk(projectId: string, input: z.infer) { + const project = await getProject({projectId}); + if (!project || project.status !== 'active') throw new Error('Project not found'); + validateLocalAssetImportCandidate({filename:input.filename,size:input.sizeBytes}); + if (input.filename.endsWith('.') || input.filename.endsWith(' ') || /^(con|prn|aux|nul|com\d|lpt\d)(\.|$)/i.test(input.filename)) + throw new Error('Invalid media filename'); + const parts = Math.ceil(input.sizeBytes / UPLOAD_CHUNK_BYTES); + if (input.partIndex >= parts) throw new Error('Upload part is outside the declared file'); + const data = Buffer.from(input.dataBase64,'base64'); + if (data.toString('base64') !== input.dataBase64) throw new Error('Invalid canonical base64'); + const expected = Math.min(UPLOAD_CHUNK_BYTES, input.sizeBytes - input.partIndex * UPLOAD_CHUNK_BYTES); + if (data.length !== expected) throw new Error('Upload part size does not match declared file'); + const projectKey = createHash('sha256').update(projectId).digest('hex'); + const directory = join(getBeatDesignDataRoot(),'mcp-uploads',projectKey,input.uploadId); + await mkdir(directory,{recursive:true}); + const meta = JSON.stringify({projectId,filename:input.filename,sizeBytes:input.sizeBytes,sha256:input.sha256}); + const manifest = join(directory,'manifest.json'); + try { await writeFile(manifest,meta,{flag:'wx'}); } + catch(error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + if (await readFile(manifest,'utf8') !== meta) throw new Error('Upload ID is already bound to a different file'); } + const receiptPath = join(directory,'receipt.json'); + const receipt = await readFile(receiptPath,'utf8').catch(error => { + if (error.code !== 'ENOENT') throw error; return null; + }); + if (receipt) return JSON.parse(receipt); + const partPath = (n:number) => join(directory,`${n}.part`); + try { await writeFile(partPath(input.partIndex),data,{flag:'wx'}); } + catch(error) { if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + if (!(await readFile(partPath(input.partIndex))).equals(data)) throw new Error('Conflicting upload part; use a new upload ID'); } + const buffers: Buffer[] = []; + const missing: number[] = []; + for (let n=0;nrm(partPath(n)))]); + return result; +} diff --git a/src/mcp/audio-generation.test.ts b/src/mcp/audio-generation.test.ts new file mode 100644 index 0000000..5d366f6 --- /dev/null +++ b/src/mcp/audio-generation.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { audioGenerationInputSchema, compileAudioGeneration } from './audio-generation'; +import { validateLocalNarration } from '@/core/generation-providers/local-drama3'; + +test('fixed voice requires a paired project Asset and exact transcript; raw paths are rejected', () => { + for (const bad of [{text:'Hi', referenceAssetId:'voice'}, {text:'Hi',referenceText:'Hello'}, + {text:'Hi',referenceAudioPath:'D:/voice.wav'}, {text:'Hi',temperature:NaN}, {text:'Hi',maxNewTokens:2048}]) + assert.equal(audioGenerationInputSchema.safeParse(bad).success, false); + const generation = compileAudioGeneration('project', {text:'Hello.',referenceAssetId:'voice',referenceText:'A reference.',maxNewTokens:512}); + assert.deepEqual(generation.references,[{assetId:'voice',role:'audio_track'}]); + assert.deepEqual(generation.parameters,{reference_text:'A reference.',max_new_tokens:512}); +}); + +test('shared provider rejects missing reference text, extra media and unknown parameters', () => { + validateLocalNarration({prompt:'Hello.'}); + validateLocalNarration({prompt:'Hello.',audio_urls:['https://validation.invalid/audio.wav'],reference_text:'A reference.',max_new_tokens:512}); + for (const input of [{prompt:'Hello.',audio_urls:['x']}, {prompt:'Hello.',reference_text:'A reference.'}, + {prompt:'Hello.',audio_urls:['a','b'],reference_text:'A reference.'}, {prompt:'Hello.',image_urls:['x']}, + {prompt:'Hello.',voice:'arbitrary'}, {prompt:'Hello.',max_new_tokens:NaN}, {prompt:'Hello.',top_p:0}]) + assert.throws(()=>validateLocalNarration(input)); +}); diff --git a/src/mcp/audio-generation.ts b/src/mcp/audio-generation.ts new file mode 100644 index 0000000..66edb9a --- /dev/null +++ b/src/mcp/audio-generation.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; +import { LOCAL_NARRATION_MAX_TEXT } from '@/core/generation-providers/local-drama3'; + +export const audioGenerationInputSchema = z.object({ + projectId: z.string().min(1).max(160).optional(), + text: z.string().trim().min(1).max(LOCAL_NARRATION_MAX_TEXT), + referenceAssetId: z.string().min(1).max(200).optional(), + referenceText: z.string().trim().min(1).max(4000).optional(), + maxNewTokens: z.number().int().min(32).max(1024).optional(), + temperature: z.number().min(0).max(2).optional(), + topP: z.number().gt(0).max(1).optional(), +}).strict().refine(v => Boolean(v.referenceAssetId) === Boolean(v.referenceText), { + message: 'referenceAssetId and its exact referenceText must be supplied together.', +}); + +export function compileAudioGeneration(projectId: string, source: z.infer) { + const input = audioGenerationInputSchema.parse(source); + return { version: 2 as const, projectId, mode: 'audio' as const, modelId: 'local-drama3', + prompt: input.text, + references: input.referenceAssetId ? [{ assetId: input.referenceAssetId, role: 'audio_track' as const }] : [], + parameters: { + ...(input.referenceText !== undefined ? { reference_text: input.referenceText } : {}), + ...(input.maxNewTokens !== undefined ? { max_new_tokens: input.maxNewTokens } : {}), + ...(input.temperature !== undefined ? { temperature: input.temperature } : {}), + ...(input.topP !== undefined ? { top_p: input.topP } : {}), + }, + }; +} diff --git a/src/mcp/server.runtime.test.ts b/src/mcp/server.runtime.test.ts index 9a851a0..13c5dc8 100644 --- a/src/mcp/server.runtime.test.ts +++ b/src/mcp/server.runtime.test.ts @@ -14,12 +14,12 @@ import { listGenerationModelDescriptors } from '@/core/generation-providers'; import { BEATDESIGN_MCP_TOOL_NAMES } from './tools'; -test('MCP tool catalog is exactly 30 named tools', () => { +test('MCP tool catalog is exactly 31 named tools', () => { const source = readFileSync(new URL('./server.ts', import.meta.url), 'utf8'); const registered = [...source.matchAll(/server\.registerTool\(\s*'([^']+)'/g)].map( (match) => match[1] ); - assert.equal(BEATDESIGN_MCP_TOOL_NAMES.length, 30); + assert.equal(BEATDESIGN_MCP_TOOL_NAMES.length, 31); assert.deepEqual(registered, [...BEATDESIGN_MCP_TOOL_NAMES]); }); diff --git a/src/mcp/server.ts b/src/mcp/server.ts index b9db443..e44b65c 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,5 +1,7 @@ import { readFile, stat } from 'node:fs/promises'; import { isAbsolute, resolve as resolvePath } from 'node:path'; +import { audioGenerationInputSchema, compileAudioGeneration } from './audio-generation'; +import { assetUploadInputSchema, uploadProjectAssetChunk } from './asset-upload'; import { createMcpHandler, @@ -684,6 +686,19 @@ export function createBeatDesignMcpServer() { }) ); + server.registerTool( + 'bdesign_asset_upload', + { + description: 'Upload user-provided media bytes as canonical base64 chunks (512 KiB decoded per part; final part may be smaller). Each uploadId is bound to a project, filename, byte length and SHA-256. Retry identical chunks safely; final assembly imports through the existing project Asset service. Maximum file size 64 MiB. Does not read arbitrary workstation paths.', + inputSchema: assetUploadInputSchema, + annotations: {destructiveHint:false,idempotentHint:true}, + }, + withToolErrors(async input => { + const project = await resolveScopedProject(input.projectId); + return uploadProjectAssetChunk(project.id,input); + }) + ); + server.registerTool( 'bdesign_asset_import', { @@ -930,26 +945,15 @@ export function createBeatDesignMcpServer() { 'bdesign_audio_generate', { description: - 'Generate short local Drama 3 narration into the current project Assets and generation history. Does not change the Canvas or Timeline. Text only; maximum 200 characters.', - inputSchema: z.object({ - projectId: idSchema.optional(), - text: z.string().min(1).max(200), - }), + 'Generate local Drama 3 narration into project Assets/history. Maximum 200 characters per take. Optionally supply one project-owned referenceAssetId with its exact referenceText to reuse a fixed voice. Does not change Canvas or Timeline.', + inputSchema: audioGenerationInputSchema, annotations: { destructiveHint: false, idempotentHint: false }, }, withToolErrors(async (input) => { const project = await resolveScopedProject(input.projectId); return submitAssetFirstGeneration({ origin: 'mcp', - generation: { - version: 2, - projectId: project.id, - mode: 'audio', - modelId: 'local-drama3', - prompt: input.text, - references: [], - parameters: {}, - }, + generation: compileAudioGeneration(project.id, input), }); }) ); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index d1f9ff1..0fd92f4 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -8,6 +8,7 @@ export const BEATDESIGN_MCP_TOOL_NAMES = [ 'bdesign_project_open', 'bdesign_asset_list', 'bdesign_asset_get', + 'bdesign_asset_upload', 'bdesign_asset_import', 'bdesign_asset_extract_frame', 'bdesign_canvas_get',