LTX-视频

LTX-Video 是一种扩散 Transformer，旨在实现从文本和图像快速实时生成高分辨率视频。LTX-Video 的主要特点是 Video-VAE。Video-VAE 具有更高的像素到潜在空间压缩比（1:192），这使得视频数据处理更高效，生成速度更快。为了支持并防止在生成过程中丢失更精细的细节，Video-VAE 解码器执行潜在空间到像素的转换**和**最后的去噪步骤。

您可以在 Lightricks 组织下找到所有原始的 LTX-Video 检查点。

以下示例演示了如何生成针对内存或推理速度进行优化的视频。

import torch
from diffusers import LTXPipeline, AutoModel
from diffusers.hooks import apply_group_offloading
from diffusers.utils import export_to_video

# fp8 layerwise weight-casting
transformer = AutoModel.from_pretrained(
    "Lightricks/LTX-Video",
    subfolder="transformer",
    torch_dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(
    storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
)

pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, torch_dtype=torch.bfloat16)

# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")

prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair.
The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek.
The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and 
natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
"""
negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)

注意事项

• 请参考 LTX-Video 仓库中推荐的生成设置。

  • Transformer、VAE 和文本编码器推荐的数据类型是 torch.bfloat16。VAE 和文本编码器也可以是 torch.float32 或 torch.float16。
  • 对于 LTX-Video 的引导蒸馏变体，将 guidance_scale 设置为 1.0。对于任何其他模型，guidance_scale 应设置得更高，例如 5.0，以获得良好的生成质量。
  • 对于时间步感知 VAE 变体（LTX-Video 0.9.1 及以上版本），将 decode_timestep 设置为 0.05，将 image_cond_noise_scale 设置为 0.025。
  • 对于支持在多个条件图像和视频之间进行插值的变体（LTX-Video 0.9.5 及以上版本），请使用相似的图像和视频以获得最佳结果。偏离条件输入可能会导致生成的视频中出现 abrupt transitionts。

• LTX-Video 0.9.7 包含一个空间潜在上采样器和一个 13B 参数的 Transformer。在推理过程中，首先快速生成低分辨率视频，然后进行上采样和精细化。

  显示示例代码

  已复制

  import torch
  from diffusers import LTXConditionPipeline, LTXLatentUpsamplePipeline
  from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
  from diffusers.utils import export_to_video, load_video
  
  pipeline = LTXConditionPipeline.from_pretrained("Lightricks/LTX-Video-0.9.7-dev", torch_dtype=torch.bfloat16)
  pipeline_upsample = LTXLatentUpsamplePipeline.from_pretrained("Lightricks/ltxv-spatial-upscaler-0.9.7", vae=pipeline.vae, torch_dtype=torch.bfloat16)
  pipeline.to("cuda")
  pipe_upsample.to("cuda")
  pipeline.vae.enable_tiling()
  
  def round_to_nearest_resolution_acceptable_by_vae(height, width):
      height = height - (height % pipeline.vae_temporal_compression_ratio)
      width = width - (width % pipeline.vae_temporal_compression_ratio)
      return height, width
  
  video = load_video(
      "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input-vid.mp4"
  )[:21]  # only use the first 21 frames as conditioning
  condition1 = LTXVideoCondition(video=video, frame_index=0)
  
  prompt = """
  The video depicts a winding mountain road covered in snow, with a single vehicle 
  traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. 
  The landscape is characterized by rugged terrain and a river visible in the distance. 
  The scene captures the solitude and beauty of a winter drive through a mountainous region.
  """
  negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
  expected_height, expected_width = 768, 1152
  downscale_factor = 2 / 3
  num_frames = 161
  
  # 1. Generate video at smaller resolution
  # Text-only conditioning is also supported without the need to pass `conditions`
  downscaled_height, downscaled_width = int(expected_height * downscale_factor), int(expected_width * downscale_factor)
  downscaled_height, downscaled_width = round_to_nearest_resolution_acceptable_by_vae(downscaled_height, downscaled_width)
  latents = pipeline(
      conditions=[condition1],
      prompt=prompt,
      negative_prompt=negative_prompt,
      width=downscaled_width,
      height=downscaled_height,
      num_frames=num_frames,
      num_inference_steps=30,
      decode_timestep=0.05,
      decode_noise_scale=0.025,
      image_cond_noise_scale=0.0,
      guidance_scale=5.0,
      guidance_rescale=0.7,
      generator=torch.Generator().manual_seed(0),
      output_type="latent",
  ).frames
  
  # 2. Upscale generated video using latent upsampler with fewer inference steps
  # The available latent upsampler upscales the height/width by 2x
  upscaled_height, upscaled_width = downscaled_height * 2, downscaled_width * 2
  upscaled_latents = pipe_upsample(
      latents=latents,
      output_type="latent"
  ).frames
  
  # 3. Denoise the upscaled video with few steps to improve texture (optional, but recommended)
  video = pipeline(
      conditions=[condition1],
      prompt=prompt,
      negative_prompt=negative_prompt,
      width=upscaled_width,
      height=upscaled_height,
      num_frames=num_frames,
      denoise_strength=0.4,  # Effectively, 4 inference steps out of 10
      num_inference_steps=10,
      latents=upscaled_latents,
      decode_timestep=0.05,
      decode_noise_scale=0.025,
      image_cond_noise_scale=0.0,
      guidance_scale=5.0,
      guidance_rescale=0.7,
      generator=torch.Generator().manual_seed(0),
      output_type="pil",
  ).frames[0]
  
  # 4. Downscale the video to the expected resolution
  video = [frame.resize((expected_width, expected_height)) for frame in video]
  
  export_to_video(video, "output.mp4", fps=24)

• LTX-Video 0.9.7 蒸馏模型经过引导和时间步蒸馏，以加速生成。它需要将 guidance_scale 设置为 1.0，并且 num_inference_steps 应设置在 4 到 10 之间以获得良好的生成质量。您还应使用以下自定义时间步以获得最佳结果。

  • 为上采样做准备的基础模型推理：[1000, 993, 987, 981, 975, 909, 725, 0.03]。
  • 上采样：[1000, 909, 725, 421, 0]。
  显示示例代码

  已复制

  import torch
  from diffusers import LTXConditionPipeline, LTXLatentUpsamplePipeline
  from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
  from diffusers.utils import export_to_video, load_video
  
  pipeline = LTXConditionPipeline.from_pretrained("Lightricks/LTX-Video-0.9.7-distilled", torch_dtype=torch.bfloat16)
  pipe_upsample = LTXLatentUpsamplePipeline.from_pretrained("Lightricks/ltxv-spatial-upscaler-0.9.7", vae=pipeline.vae, torch_dtype=torch.bfloat16)
  pipeline.to("cuda")
  pipe_upsample.to("cuda")
  pipeline.vae.enable_tiling()
  
  def round_to_nearest_resolution_acceptable_by_vae(height, width):
      height = height - (height % pipeline.vae_temporal_compression_ratio)
      width = width - (width % pipeline.vae_temporal_compression_ratio)
      return height, width
  
  prompt = """
  artistic anatomical 3d render, utlra quality, human half full male body with transparent 
  skin revealing structure instead of organs, muscular, intricate creative patterns, 
  monochromatic with backlighting, lightning mesh, scientific concept art, blending biology 
  with botany, surreal and ethereal quality, unreal engine 5, ray tracing, ultra realistic, 
  16K UHD, rich details. camera zooms out in a rotating fashion
  """
  negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
  expected_height, expected_width = 768, 1152
  downscale_factor = 2 / 3
  num_frames = 161
  
  # 1. Generate video at smaller resolution
  downscaled_height, downscaled_width = int(expected_height * downscale_factor), int(expected_width * downscale_factor)
  downscaled_height, downscaled_width = round_to_nearest_resolution_acceptable_by_vae(downscaled_height, downscaled_width)
  latents = pipeline(
      prompt=prompt,
      negative_prompt=negative_prompt,
      width=downscaled_width,
      height=downscaled_height,
      num_frames=num_frames,
      timesteps=[1000, 993, 987, 981, 975, 909, 725, 0.03],
      decode_timestep=0.05,
      decode_noise_scale=0.025,
      image_cond_noise_scale=0.0,
      guidance_scale=1.0,
      guidance_rescale=0.7,
      generator=torch.Generator().manual_seed(0),
      output_type="latent",
  ).frames
  
  # 2. Upscale generated video using latent upsampler with fewer inference steps
  # The available latent upsampler upscales the height/width by 2x
  upscaled_height, upscaled_width = downscaled_height * 2, downscaled_width * 2
  upscaled_latents = pipe_upsample(
      latents=latents,
      adain_factor=1.0,
      output_type="latent"
  ).frames
  
  # 3. Denoise the upscaled video with few steps to improve texture (optional, but recommended)
  video = pipeline(
      prompt=prompt,
      negative_prompt=negative_prompt,
      width=upscaled_width,
      height=upscaled_height,
      num_frames=num_frames,
      denoise_strength=0.999,  # Effectively, 4 inference steps out of 5
      timesteps=[1000, 909, 725, 421, 0],
      latents=upscaled_latents,
      decode_timestep=0.05,
      decode_noise_scale=0.025,
      image_cond_noise_scale=0.0,
      guidance_scale=1.0,
      guidance_rescale=0.7,
      generator=torch.Generator().manual_seed(0),
      output_type="pil",
  ).frames[0]
  
  # 4. Downscale the video to the expected resolution
  video = [frame.resize((expected_width, expected_height)) for frame in video]
  
  export_to_video(video, "output.mp4", fps=24)

• LTX-Video 支持使用 load_lora_weights() 加载 LoRA。

  显示示例代码

  已复制

  import torch
  from diffusers import LTXConditionPipeline
  from diffusers.utils import export_to_video, load_image
  
  pipeline = LTXConditionPipeline.from_pretrained(
      "Lightricks/LTX-Video-0.9.5", torch_dtype=torch.bfloat16
  )
  
  pipeline.load_lora_weights("Lightricks/LTX-Video-Cakeify-LoRA", adapter_name="cakeify")
  pipeline.set_adapters("cakeify")
  
  # use "CAKEIFY" to trigger the LoRA
  prompt = "CAKEIFY a person using a knife to cut a cake shaped like a Pikachu plushie"
  image = load_image("https://huggingface.co/Lightricks/LTX-Video-Cakeify-LoRA/resolve/main/assets/images/pikachu.png")
  
  video = pipeline(
      prompt=prompt,
      image=image,
      width=576,
      height=576,
      num_frames=161,
      decode_timestep=0.03,
      decode_noise_scale=0.025,
      num_inference_steps=50,
  ).frames[0]
  export_to_video(video, "output.mp4", fps=26)

• LTX-Video 支持从单个文件加载，例如 GGUF 检查点，使用 loaders.FromOriginalModelMixin.from_single_file() 或 loaders.FromSingleFileMixin.from_single_file()。

  显示示例代码

  已复制

  import torch
  from diffusers.utils import export_to_video
  from diffusers import LTXPipeline, AutoModel, GGUFQuantizationConfig
  
  transformer = AutoModel.from_single_file(
      "https://huggingface.co/city96/LTX-Video-gguf/blob/main/ltx-video-2b-v0.9-Q3_K_S.gguf"
      quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
      torch_dtype=torch.bfloat16
  )
  pipeline = LTXPipeline.from_pretrained(
      "Lightricks/LTX-Video",
      transformer=transformer,
      torch_dtype=torch.bfloat16
  )

LTXPipeline

>>> import torch
>>> from diffusers import LTXPipeline
>>> from diffusers.utils import export_to_video

>>> pipe = LTXPipeline.from_pretrained("Lightricks/LTX-Video", torch_dtype=torch.bfloat16)
>>> pipe.to("cuda")

>>> prompt = "A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage"
>>> negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

>>> video = pipe(
...     prompt=prompt,
...     negative_prompt=negative_prompt,
...     width=704,
...     height=480,
...     num_frames=161,
...     num_inference_steps=50,
... ).frames[0]
>>> export_to_video(video, "output.mp4", fps=24)

LTXImageToVideoPipeline

>>> import torch
>>> from diffusers import LTXImageToVideoPipeline
>>> from diffusers.utils import export_to_video, load_image

>>> pipe = LTXImageToVideoPipeline.from_pretrained("Lightricks/LTX-Video", torch_dtype=torch.bfloat16)
>>> pipe.to("cuda")

>>> image = load_image(
...     "https://huggingface.co/datasets/a-r-r-o-w/tiny-meme-dataset-captioned/resolve/main/images/8.png"
... )
>>> prompt = "A young girl stands calmly in the foreground, looking directly at the camera, as a house fire rages in the background. Flames engulf the structure, with smoke billowing into the air. Firefighters in protective gear rush to the scene, a fire truck labeled '38' visible behind them. The girl's neutral expression contrasts sharply with the chaos of the fire, creating a poignant and emotionally charged scene."
>>> negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

>>> video = pipe(
...     image=image,
...     prompt=prompt,
...     negative_prompt=negative_prompt,
...     width=704,
...     height=480,
...     num_frames=161,
...     num_inference_steps=50,
... ).frames[0]
>>> export_to_video(video, "output.mp4", fps=24)

LTXConditionPipeline

>>> import torch
>>> from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXConditionPipeline, LTXVideoCondition
>>> from diffusers.utils import export_to_video, load_video, load_image

>>> pipe = LTXConditionPipeline.from_pretrained("Lightricks/LTX-Video-0.9.5", torch_dtype=torch.bfloat16)
>>> pipe.to("cuda")

>>> # Load input image and video
>>> video = load_video(
...     "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input-vid.mp4"
... )
>>> image = load_image(
...     "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cosmos/cosmos-video2world-input.jpg"
... )

>>> # Create conditioning objects
>>> condition1 = LTXVideoCondition(
...     image=image,
...     frame_index=0,
... )
>>> condition2 = LTXVideoCondition(
...     video=video,
...     frame_index=80,
... )

>>> prompt = "The video depicts a long, straight highway stretching into the distance, flanked by metal guardrails. The road is divided into multiple lanes, with a few vehicles visible in the far distance. The surrounding landscape features dry, grassy fields on one side and rolling hills on the other. The sky is mostly clear with a few scattered clouds, suggesting a bright, sunny day. And then the camera switch to a winding mountain road covered in snow, with a single vehicle traveling along it. The road is flanked by steep, rocky cliffs and sparse vegetation. The landscape is characterized by rugged terrain and a river visible in the distance. The scene captures the solitude and beauty of a winter drive through a mountainous region."
>>> negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

>>> # Generate video
>>> generator = torch.Generator("cuda").manual_seed(0)
>>> # Text-only conditioning is also supported without the need to pass `conditions`
>>> video = pipe(
...     conditions=[condition1, condition2],
...     prompt=prompt,
...     negative_prompt=negative_prompt,
...     width=768,
...     height=512,
...     num_frames=161,
...     num_inference_steps=40,
...     generator=generator,
... ).frames[0]

>>> export_to_video(video, "output.mp4", fps=24)

LTXLatentUpsamplePipeline

LTXPipelineOutput