[feat] Add JoyAI-Echo multi-shot audio-video generation pipeline

docs/source/en/_toctree.yml

@@ -549,6 +549,8 @@
         title: Ideogram 4
       - local: api/pipelines/pix2pix
         title: InstructPix2Pix
+      - local: api/pipelines/joyai_echo
+        title: JoyAI Echo
       - local: api/pipelines/joyimage_edit
         title: JoyImage Edit
       - local: api/pipelines/kandinsky

docs/source/en/api/pipelines/joyai_echo.md

@@ -0,0 +1,43 @@
+# JoyAI-Echo
+
+JoyAI-Echo is a text-to-audio-video generation pipeline for multi-shot video stories. It builds on the LTX-2 component
+layout and adds the JoyAI-Echo few-step DMD denoising schedule plus a paired audio-video memory bank for cross-shot
+consistency.
+
+The pipeline accepts one prompt per shot. When a list of prompts is passed, generated video and audio latents from
+earlier shots are kept as memory tokens for later shots.
+
+```py
+import torch
+from diffusers import JoyAIEchoPipeline
+from diffusers.utils import encode_video
+
+pipe = JoyAIEchoPipeline.from_pretrained("path/to/converted-joyai-echo", torch_dtype=torch.bfloat16)
+pipe.enable_model_cpu_offload()
+
+output = pipe(
+    [
+        "A cinematic opening shot of the protagonist entering a quiet train station.",
+        "The same protagonist speaks softly while the camera follows through the platform.",
+    ],
+    height=736,
+    width=1280,
+    num_frames=241,
+    frame_rate=25.0,
+)
+
+for i, (frames, audio) in enumerate(zip(output.frames, output.audio)):
+    encode_video(frames[0], fps=25, audio=audio[0].float().cpu(), output_path=f"shot_{i:03d}.mp4")
+```
+
+## JoyAIEchoPipeline
+
+[[autodoc]] JoyAIEchoPipeline
+
+## JoyAIEchoPipelineOutput
+
+[[autodoc]] pipelines.joyai_echo.JoyAIEchoPipelineOutput
+
+## JoyAIEchoShotOutput
+
+[[autodoc]] pipelines.joyai_echo.JoyAIEchoShotOutput

src/diffusers/__init__.py

@@ -272,6 +272,7 @@
             "HunyuanVideoTransformer3DModel",
             "I2VGenXLUNet",
             "Ideogram4Transformer2DModel",
+            "JoyAIEchoTransformer3DModel",
             "JoyImageEditTransformer3DModel",
             "Kandinsky3UNet",
             "Kandinsky5Transformer3DModel",
@@ -616,6 +617,10 @@
             "IFPipeline",
             "IFSuperResolutionPipeline",
             "ImageTextPipelineOutput",
+            "JoyAIEchoOriginalCheckpointPipeline",
+            "JoyAIEchoPipeline",
+            "JoyAIEchoPipelineOutput",
+            "JoyAIEchoShotOutput",
             "JoyImageEditPipeline",
             "JoyImageEditPipelineOutput",
             "Kandinsky3Img2ImgPipeline",
@@ -1126,6 +1131,7 @@
             HunyuanVideoTransformer3DModel,
             I2VGenXLUNet,
             Ideogram4Transformer2DModel,
+            JoyAIEchoTransformer3DModel,
             JoyImageEditTransformer3DModel,
             Kandinsky3UNet,
             Kandinsky5Transformer3DModel,
@@ -1445,6 +1451,10 @@
             IFPipeline,
             IFSuperResolutionPipeline,
             ImageTextPipelineOutput,
+            JoyAIEchoOriginalCheckpointPipeline,
+            JoyAIEchoPipeline,
+            JoyAIEchoPipelineOutput,
+            JoyAIEchoShotOutput,
             JoyImageEditPipeline,
             JoyImageEditPipelineOutput,
             Kandinsky3Img2ImgPipeline,

src/diffusers/models/__init__.py

@@ -119,6 +119,7 @@
     _import_structure["transformers.transformer_hunyuan_video_framepack"] = ["HunyuanVideoFramepackTransformer3DModel"]
     _import_structure["transformers.transformer_hunyuanimage"] = ["HunyuanImageTransformer2DModel"]
     _import_structure["transformers.transformer_ideogram4"] = ["Ideogram4Transformer2DModel"]
+    _import_structure["transformers.transformer_joyai_echo"] = ["JoyAIEchoTransformer3DModel"]
     _import_structure["transformers.transformer_joyimage"] = ["JoyImageEditTransformer3DModel"]
     _import_structure["transformers.transformer_kandinsky"] = ["Kandinsky5Transformer3DModel"]
     _import_structure["transformers.transformer_longcat_audio_dit"] = ["LongCatAudioDiTTransformer"]
@@ -250,6 +251,7 @@
             HunyuanVideoFramepackTransformer3DModel,
             HunyuanVideoTransformer3DModel,
             Ideogram4Transformer2DModel,
+            JoyAIEchoTransformer3DModel,
             JoyImageEditTransformer3DModel,
             Kandinsky5Transformer3DModel,
             LatteTransformer3DModel,

src/diffusers/models/transformers/__init__.py

@@ -40,6 +40,7 @@
     from .transformer_hunyuan_video_framepack import HunyuanVideoFramepackTransformer3DModel
     from .transformer_hunyuanimage import HunyuanImageTransformer2DModel
     from .transformer_ideogram4 import Ideogram4Transformer2DModel
+    from .transformer_joyai_echo import JoyAIEchoTransformer3DModel
     from .transformer_joyimage import JoyImageEditTransformer3DModel
     from .transformer_kandinsky import Kandinsky5Transformer3DModel
     from .transformer_longcat_audio_dit import LongCatAudioDiTTransformer