dagloop5 commited on
Commit
30947a4
·
verified ·
1 Parent(s): 6fee77f

Delete app(wipwithneg).py

Browse files
Files changed (1) hide show
  1. app(wipwithneg).py +0 -861
app(wipwithneg).py DELETED
@@ -1,861 +0,0 @@
1
- import os
2
- import subprocess
3
- import sys
4
-
5
- # Disable torch.compile / dynamo before any torch import
6
- os.environ["TORCH_COMPILE_DISABLE"] = "1"
7
- os.environ["TORCHDYNAMO_DISABLE"] = "1"
8
-
9
- # Install xformers for memory-efficient attention
10
- subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
11
-
12
- # Clone LTX-2 repo and install packages
13
- LTX_REPO_URL = "https://github.com/Lightricks/LTX-2.git"
14
- LTX_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "LTX-2")
15
-
16
- LTX_COMMIT = "a2c3f24078eb918171967f74b6f66b756b29ee45" # known working commit with decode_video
17
-
18
- if not os.path.exists(LTX_REPO_DIR):
19
- print(f"Cloning {LTX_REPO_URL}...")
20
- subprocess.run(["git", "clone", LTX_REPO_URL, LTX_REPO_DIR], check=True)
21
-
22
- print(f"Checking out pinned commit {LTX_COMMIT}...")
23
- subprocess.run(["git", "fetch", "--all", "--tags"], cwd=LTX_REPO_DIR, check=True)
24
- subprocess.run(["git", "checkout", LTX_COMMIT], cwd=LTX_REPO_DIR, check=True)
25
-
26
- print("Installing ltx-core and ltx-pipelines from cloned repo...")
27
- subprocess.run(
28
- [sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", "-e",
29
- os.path.join(LTX_REPO_DIR, "packages", "ltx-core"),
30
- "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines")],
31
- check=True,
32
- )
33
-
34
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
35
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
36
-
37
- import logging
38
- import random
39
- import tempfile
40
- from pathlib import Path
41
- import gc
42
- import hashlib
43
-
44
- import torch
45
- torch._dynamo.config.suppress_errors = True
46
- torch._dynamo.config.disable = True
47
-
48
- import spaces
49
- import gradio as gr
50
- import numpy as np
51
- from huggingface_hub import hf_hub_download, snapshot_download
52
- from safetensors.torch import load_file, save_file
53
- from safetensors import safe_open
54
- import json
55
- import requests
56
-
57
- from ltx_core.components.diffusion_steps import Res2sDiffusionStep
58
- from ltx_core.components.guiders import MultiModalGuider, MultiModalGuiderParams
59
- from ltx_core.components.noisers import GaussianNoiser
60
- from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number
61
- from ltx_core.types import Audio, VideoLatentShape, VideoPixelShape
62
- from ltx_pipelines.utils.args import ImageConditioningInput, hq_2_stage_arg_parser
63
- from ltx_pipelines.utils.blocks import (
64
- AudioDecoder,
65
- DiffusionStage,
66
- ImageConditioner,
67
- PromptEncoder,
68
- VideoDecoder,
69
- VideoUpsampler,
70
- )
71
- from ltx_pipelines.utils.constants import LTX_2_3_HQ_PARAMS, STAGE_2_DISTILLED_SIGMAS
72
- from ltx_pipelines.utils.denoisers import GuidedDenoiser, SimpleDenoiser
73
- from ltx_pipelines.utils.helpers import (
74
- assert_resolution,
75
- combined_image_conditionings,
76
- get_device,
77
- )
78
- from ltx_pipelines.utils.media_io import encode_video
79
- from ltx_pipelines.utils.samplers import res2s_audio_video_denoising_loop
80
- from ltx_core.loader.primitives import LoraPathStrengthAndSDOps
81
- from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP
82
-
83
- from collections.abc import Iterator
84
-
85
- from ltx_core.components.schedulers import LTX2Scheduler
86
- from ltx_core.loader.registry import Registry
87
- from ltx_core.quantization import QuantizationPolicy
88
- from ltx_pipelines.utils.types import ModalitySpec
89
-
90
- # Force-patch xformers attention into the LTX attention module.
91
- from ltx_core.model.transformer import attention as _attn_mod
92
- print(f"[ATTN] Before patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
93
- try:
94
- from xformers.ops import memory_efficient_attention as _mea
95
- _attn_mod.memory_efficient_attention = _mea
96
- print(f"[ATTN] After patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
97
- except Exception as e:
98
- print(f"[ATTN] xformers patch FAILED: {type(e).__name__}: {e}")
99
-
100
- logging.getLogger().setLevel(logging.INFO)
101
-
102
- MAX_SEED = np.iinfo(np.int32).max
103
- DEFAULT_PROMPT = (
104
- "An astronaut hatches from a fragile egg on the surface of the Moon, "
105
- "the shell cracking and peeling apart in gentle low-gravity motion. "
106
- "Fine lunar dust lifts and drifts outward with each movement, floating "
107
- "in slow arcs before settling back onto the ground."
108
- )
109
- DEFAULT_FRAME_RATE = 24.0
110
-
111
- # Resolution presets: (width, height)
112
- RESOLUTIONS = {
113
- "high": {"16:9": (1536, 1024), "9:16": (1024, 1536), "1:1": (1024, 1024)},
114
- "low": {"16:9": (768, 512), "9:16": (512, 768), "1:1": (768, 768)},
115
- }
116
-
117
-
118
- class LTX23NegativePromptTwoStagePipeline:
119
- def __init__(
120
- self,
121
- checkpoint_path: str,
122
- spatial_upsampler_path: str,
123
- gemma_root: str,
124
- loras: tuple[LoraPathStrengthAndSDOps, ...],
125
- device: torch.device | None = None,
126
- quantization: QuantizationPolicy | None = None,
127
- registry: Registry | None = None,
128
- torch_compile: bool = False,
129
- ):
130
- self.device = device or get_device()
131
- self.dtype = torch.bfloat16
132
- self._scheduler = LTX2Scheduler()
133
-
134
- self.prompt_encoder = PromptEncoder(checkpoint_path, gemma_root, self.dtype, self.device, registry=registry)
135
- self.image_conditioner = ImageConditioner(checkpoint_path, self.dtype, self.device, registry=registry)
136
- self.upsampler = VideoUpsampler(checkpoint_path, spatial_upsampler_path, self.dtype, self.device, registry=registry)
137
- self.video_decoder = VideoDecoder(checkpoint_path, self.dtype, self.device, registry=registry)
138
- self.audio_decoder = AudioDecoder(checkpoint_path, self.dtype, self.device, registry=registry)
139
-
140
- self.stage_1 = DiffusionStage(
141
- checkpoint_path,
142
- self.dtype,
143
- self.device,
144
- loras=tuple(loras),
145
- quantization=quantization,
146
- registry=registry,
147
- torch_compile=torch_compile,
148
- )
149
- self.stage_2 = DiffusionStage(
150
- checkpoint_path,
151
- self.dtype,
152
- self.device,
153
- loras=tuple(loras),
154
- quantization=quantization,
155
- registry=registry,
156
- torch_compile=torch_compile,
157
- )
158
-
159
- def __call__(
160
- self,
161
- prompt: str,
162
- negative_prompt: str,
163
- seed: int,
164
- height: int,
165
- width: int,
166
- num_frames: int,
167
- frame_rate: float,
168
- images: list[ImageConditioningInput],
169
- tiling_config: TilingConfig | None = None,
170
- enhance_prompt: bool = False,
171
- streaming_prefetch_count: int | None = None,
172
- max_batch_size: int = 1,
173
- num_inference_steps: int = 8,
174
- stage_1_sigmas: torch.Tensor | None = None,
175
- stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS,
176
- video_guider_params: MultiModalGuiderParams | None = None,
177
- audio_guider_params: MultiModalGuiderParams | None = None,
178
- ) -> tuple[Iterator[torch.Tensor], Audio]:
179
- assert_resolution(height=height, width=width, is_two_stage=True)
180
-
181
- generator = torch.Generator(device=self.device).manual_seed(seed)
182
- noiser = GaussianNoiser(generator=generator)
183
- dtype = torch.bfloat16
184
-
185
- ctx_p, ctx_n = self.prompt_encoder(
186
- [prompt, negative_prompt],
187
- enhance_first_prompt=enhance_prompt,
188
- enhance_prompt_image=(
189
- __import__('PIL.Image', fromlist=['Image']).open(images[0].path)
190
- if (len(images) > 0 and enhance_prompt) else None
191
- ),
192
- enhance_prompt_seed=seed,
193
- streaming_prefetch_count=streaming_prefetch_count,
194
- )
195
- v_context_p, a_context_p = ctx_p.video_encoding, ctx_p.audio_encoding
196
- v_context_n, a_context_n = ctx_n.video_encoding, ctx_n.audio_encoding
197
-
198
- if video_guider_params is None:
199
- video_guider_params = LTX_2_3_HQ_PARAMS.video_guider_params
200
-
201
- if audio_guider_params is None:
202
- audio_guider_params = LTX_2_3_HQ_PARAMS.audio_guider_params
203
-
204
- stage_1_output_shape = VideoPixelShape(
205
- batch=1, frames=num_frames, width=width // 2, height=height // 2, fps=frame_rate
206
- )
207
- stage_1_conditionings = self.image_conditioner(
208
- lambda enc: combined_image_conditionings(
209
- images=images,
210
- height=stage_1_output_shape.height,
211
- width=stage_1_output_shape.width,
212
- video_encoder=enc,
213
- dtype=dtype,
214
- device=self.device,
215
- )
216
- )
217
-
218
- stepper = Res2sDiffusionStep()
219
- if stage_1_sigmas is None:
220
- empty_latent = torch.empty(VideoLatentShape.from_pixel_shape(stage_1_output_shape).to_torch_shape())
221
- stage_1_sigmas = self._scheduler.execute(latent=empty_latent, steps=num_inference_steps)
222
- sigmas = stage_1_sigmas.to(dtype=torch.float32, device=self.device)
223
-
224
- video_state, audio_state = self.stage_1(
225
- denoiser=GuidedDenoiser(
226
- v_context=v_context_p,
227
- a_context=a_context_p,
228
- video_guider=MultiModalGuider(
229
- params=video_guider_params,
230
- negative_context=v_context_n,
231
- ),
232
- audio_guider=MultiModalGuider(
233
- params=audio_guider_params,
234
- negative_context=a_context_n,
235
- ),
236
- ),
237
- sigmas=sigmas,
238
- noiser=noiser,
239
- stepper=stepper,
240
- width=stage_1_output_shape.width,
241
- height=stage_1_output_shape.height,
242
- frames=num_frames,
243
- fps=frame_rate,
244
- video=ModalitySpec(context=v_context_p, conditionings=stage_1_conditionings),
245
- audio=ModalitySpec(context=a_context_p),
246
- loop=res2s_audio_video_denoising_loop,
247
- streaming_prefetch_count=streaming_prefetch_count,
248
- max_batch_size=max_batch_size,
249
- )
250
-
251
- upscaled_video_latent = self.upsampler(video_state.latent[:1])
252
-
253
- stage_2_conditionings = self.image_conditioner(
254
- lambda enc: combined_image_conditionings(
255
- images=images,
256
- height=height,
257
- width=width,
258
- video_encoder=enc,
259
- dtype=dtype,
260
- device=self.device,
261
- )
262
- )
263
-
264
- video_state, audio_state = self.stage_2(
265
- denoiser=SimpleDenoiser(v_context=v_context_p, a_context=a_context_p),
266
- sigmas=stage_2_sigmas.to(dtype=torch.float32, device=self.device),
267
- noiser=noiser,
268
- stepper=stepper,
269
- width=width,
270
- height=height,
271
- frames=num_frames,
272
- fps=frame_rate,
273
- video=ModalitySpec(
274
- context=v_context_p,
275
- conditionings=stage_2_conditionings,
276
- noise_scale=stage_2_sigmas[0].item(),
277
- initial_latent=upscaled_video_latent,
278
- ),
279
- audio=ModalitySpec(
280
- context=a_context_p,
281
- noise_scale=stage_2_sigmas[0].item(),
282
- initial_latent=audio_state.latent,
283
- ),
284
- loop=res2s_audio_video_denoising_loop,
285
- streaming_prefetch_count=streaming_prefetch_count,
286
- )
287
-
288
- decoded_video = self.video_decoder(video_state.latent, tiling_config, generator)
289
- decoded_audio = self.audio_decoder(audio_state.latent)
290
- return decoded_video, decoded_audio
291
-
292
-
293
- # Model repos
294
- LTX_MODEL_REPO = "Lightricks/LTX-2.3"
295
- GEMMA_REPO ="Lightricks/gemma-3-12b-it-qat-q4_0-unquantized"
296
-
297
- # Download model checkpoints
298
- print("=" * 80)
299
- print("Downloading LTX-2.3 distilled model + Gemma...")
300
- print("=" * 80)
301
-
302
- # LoRA cache directory and currently-applied key
303
- LORA_CACHE_DIR = Path("lora_cache")
304
- LORA_CACHE_DIR.mkdir(exist_ok=True)
305
-
306
- current_lora_key: str | None = None
307
- PENDING_LORA_KEY: str | None = None
308
- PENDING_LORA_LORAS: tuple[LoraPathStrengthAndSDOps, ...] | None = None
309
- PENDING_LORA_STATUS: str = "No LoRA config prepared yet."
310
-
311
- weights_dir = Path("weights")
312
- weights_dir.mkdir(exist_ok=True)
313
- checkpoint_path = hf_hub_download(
314
- repo_id=LTX_MODEL_REPO,
315
- filename="ltx-2.3-22b-distilled-1.1.safetensors",
316
- local_dir=str(weights_dir),
317
- local_dir_use_symlinks=False,
318
- )
319
- spatial_upsampler_path = hf_hub_download(repo_id=LTX_MODEL_REPO, filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors")
320
- gemma_root = snapshot_download(repo_id=GEMMA_REPO)
321
-
322
- # ---- Insert block (LoRA downloads) between lines 268 and 269 ----
323
- # LoRA repo + download the requested LoRA adapters
324
- LORA_REPO = "dagloop5/LoRA"
325
-
326
- print("=" * 80)
327
- print("Downloading LoRA adapters from dagloop5/LoRA...")
328
- print("=" * 80)
329
- pose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2_3_NSFW_furry_concat_v2.safetensors")
330
- general_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_reasoning_I2V_V3.safetensors")
331
- motion_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="motion_helper.safetensors")
332
- dreamlay_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="DR34ML4Y_LTXXX_PREVIEW_RC1.safetensors") # m15510n4ry, bl0wj0b, d0ubl3_bj, d0gg1e, c0wg1rl
333
- mself_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="Furry Hyper Masturbation - LTX-2 I2V v1.safetensors") # Hyperfap
334
- dramatic_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2.3 - Orgasm.safetensors") # "[He | She] is having am orgasm." (am or an?)
335
- fluid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="cr3ampi3_animation_i2v_ltx2_v1.0.safetensors") # cr3ampi3 animation., missionary animation, doggystyle bouncy animation, double penetration animation
336
- liquid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="liquid_wet_dr1pp_ltx2_v1.0_scaled.safetensors") # wet dr1pp
337
- demopose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="clapping-cheeks-audio-v001-alpha.safetensors")
338
- voice_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="hentai_voice_ltx23.safetensors")
339
- realism_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="FurryenhancerLTX2.3V1.215.safetensors")
340
- transition_lora_path = hf_hub_download(repo_id="valiantcat/LTX-2.3-Transition-LORA", filename="ltx2.3-transition.safetensors")
341
-
342
- print(f"Pose LoRA: {pose_lora_path}")
343
- print(f"General LoRA: {general_lora_path}")
344
- print(f"Motion LoRA: {motion_lora_path}")
345
- print(f"Dreamlay LoRA: {dreamlay_lora_path}")
346
- print(f"Mself LoRA: {mself_lora_path}")
347
- print(f"Dramatic LoRA: {dramatic_lora_path}")
348
- print(f"Fluid LoRA: {fluid_lora_path}")
349
- print(f"Liquid LoRA: {liquid_lora_path}")
350
- print(f"Demopose LoRA: {demopose_lora_path}")
351
- print(f"Voice LoRA: {voice_lora_path}")
352
- print(f"Realism LoRA: {realism_lora_path}")
353
- print(f"Transition LoRA: {transition_lora_path}")
354
- # ----------------------------------------------------------------
355
-
356
- print(f"Checkpoint: {checkpoint_path}")
357
- print(f"Spatial upsampler: {spatial_upsampler_path}")
358
- print(f"Gemma root: {gemma_root}")
359
-
360
- # Initialize pipeline WITH text encoder and optional audio support
361
- # ---- Replace block (pipeline init) lines 275-281 ----
362
- pipeline = LTX23NegativePromptTwoStagePipeline(
363
- checkpoint_path=str(checkpoint_path),
364
- spatial_upsampler_path=str(spatial_upsampler_path),
365
- gemma_root=str(gemma_root),
366
- loras=[],
367
- quantization=QuantizationPolicy.fp8_cast(),
368
- )
369
- # ----------------------------------------------------------------
370
-
371
- def _make_lora_key(pose_strength: float, general_strength: float, motion_strength: float, dreamlay_strength: float, mself_strength: float, dramatic_strength: float, fluid_strength: float, liquid_strength: float, demopose_strength: float, voice_strength: float, realism_strength: float, transition_strength: float) -> tuple[str, str]:
372
- rp = round(float(pose_strength), 2)
373
- rg = round(float(general_strength), 2)
374
- rm = round(float(motion_strength), 2)
375
- rd = round(float(dreamlay_strength), 2)
376
- rs = round(float(mself_strength), 2)
377
- rr = round(float(dramatic_strength), 2)
378
- rf = round(float(fluid_strength), 2)
379
- rl = round(float(liquid_strength), 2)
380
- ro = round(float(demopose_strength), 2)
381
- rv = round(float(voice_strength), 2)
382
- re = round(float(realism_strength), 2)
383
- rt = round(float(transition_strength), 2)
384
- key_str = f"{pose_lora_path}:{rp}|{general_lora_path}:{rg}|{motion_lora_path}:{rm}|{dreamlay_lora_path}:{rd}|{mself_lora_path}:{rs}|{dramatic_lora_path}:{rr}|{fluid_lora_path}:{rf}|{liquid_lora_path}:{rl}|{demopose_lora_path}:{ro}|{voice_lora_path}:{rv}|{realism_lora_path}:{re}|{transition_lora_path}:{rt}"
385
- key = hashlib.sha256(key_str.encode("utf-8")).hexdigest()
386
- return key
387
-
388
-
389
- def prepare_lora_cache(
390
- pose_strength: float,
391
- general_strength: float,
392
- motion_strength: float,
393
- dreamlay_strength: float,
394
- mself_strength: float,
395
- dramatic_strength: float,
396
- fluid_strength: float,
397
- liquid_strength: float,
398
- demopose_strength: float,
399
- voice_strength: float,
400
- realism_strength: float,
401
- transition_strength: float,
402
- progress=gr.Progress(track_tqdm=True),
403
- ):
404
- """
405
- Prepare the LoRA selection for the guided pipeline.
406
- This caches the LoRA config, not fused weights.
407
- """
408
- global PENDING_LORA_KEY, PENDING_LORA_LORAS, PENDING_LORA_STATUS
409
-
410
- key = _make_lora_key(
411
- pose_strength, general_strength, motion_strength, dreamlay_strength,
412
- mself_strength, dramatic_strength, fluid_strength, liquid_strength,
413
- demopose_strength, voice_strength, realism_strength, transition_strength
414
- )
415
- cache_path = LORA_CACHE_DIR / f"{key}.json"
416
-
417
- progress(0.05, desc="Preparing LoRA config")
418
-
419
- entries = [
420
- (pose_lora_path, round(float(pose_strength), 2)),
421
- (general_lora_path, round(float(general_strength), 2)),
422
- (motion_lora_path, round(float(motion_strength), 2)),
423
- (dreamlay_lora_path, round(float(dreamlay_strength), 2)),
424
- (mself_lora_path, round(float(mself_strength), 2)),
425
- (dramatic_lora_path, round(float(dramatic_strength), 2)),
426
- (fluid_lora_path, round(float(fluid_strength), 2)),
427
- (liquid_lora_path, round(float(liquid_strength), 2)),
428
- (demopose_lora_path, round(float(demopose_strength), 2)),
429
- (voice_lora_path, round(float(voice_strength), 2)),
430
- (realism_lora_path, round(float(realism_strength), 2)),
431
- (transition_lora_path, round(float(transition_strength), 2)),
432
- ]
433
-
434
- loras_for_builder = [
435
- LoraPathStrengthAndSDOps(path, strength, LTXV_LORA_COMFY_RENAMING_MAP)
436
- for path, strength in entries
437
- if path is not None and float(strength) != 0.0
438
- ]
439
-
440
- if not loras_for_builder:
441
- PENDING_LORA_KEY = None
442
- PENDING_LORA_LORAS = None
443
- PENDING_LORA_STATUS = "No non-zero LoRA strengths selected; nothing to prepare."
444
- return PENDING_LORA_STATUS
445
-
446
- try:
447
- if cache_path.exists():
448
- progress(0.20, desc="Loading cached LoRA config")
449
- data = json.loads(cache_path.read_text())
450
- loras_for_builder = [
451
- LoraPathStrengthAndSDOps(item["path"], item["strength"], LTXV_LORA_COMFY_RENAMING_MAP)
452
- for item in data
453
- if float(item["strength"]) != 0.0
454
- ]
455
- else:
456
- progress(0.30, desc="Saving LoRA config cache")
457
- cache_path.write_text(
458
- json.dumps(
459
- [{"path": path, "strength": strength} for path, strength in entries if float(strength) != 0.0],
460
- indent=2,
461
- )
462
- )
463
-
464
- PENDING_LORA_KEY = key
465
- PENDING_LORA_LORAS = tuple(loras_for_builder)
466
- PENDING_LORA_STATUS = f"Prepared LoRA config: {cache_path.name}"
467
- return PENDING_LORA_STATUS
468
-
469
- except Exception as e:
470
- import traceback
471
- print(f"[LoRA] Prepare failed: {type(e).__name__}: {e}")
472
- print(traceback.format_exc())
473
- PENDING_LORA_KEY = None
474
- PENDING_LORA_LORAS = None
475
- PENDING_LORA_STATUS = f"LoRA prepare failed: {type(e).__name__}: {e}"
476
- return PENDING_LORA_STATUS
477
-
478
-
479
- def apply_prepared_lora_config_to_pipeline():
480
- global current_lora_key, PENDING_LORA_KEY, PENDING_LORA_LORAS, pipeline
481
-
482
- if PENDING_LORA_LORAS is None or PENDING_LORA_KEY is None:
483
- print("[LoRA] No prepared LoRA config available; skipping.")
484
- return False
485
-
486
- if current_lora_key == PENDING_LORA_KEY:
487
- print("[LoRA] Prepared LoRA config already active; skipping.")
488
- return True
489
-
490
- del pipeline
491
- gc.collect()
492
- torch.cuda.empty_cache()
493
-
494
- pipeline = LTX23NegativePromptTwoStagePipeline(
495
- checkpoint_path=str(checkpoint_path),
496
- spatial_upsampler_path=str(spatial_upsampler_path),
497
- gemma_root=str(gemma_root),
498
- loras=PENDING_LORA_LORAS,
499
- quantization=QuantizationPolicy.fp8_cast(),
500
- )
501
-
502
- current_lora_key = PENDING_LORA_KEY
503
- print("[LoRA] Prepared LoRA config applied by rebuilding the pipeline.")
504
- return True
505
-
506
- print("=" * 80)
507
- print("Pipeline ready!")
508
- print("=" * 80)
509
-
510
-
511
- def log_memory(tag: str):
512
- if torch.cuda.is_available():
513
- allocated = torch.cuda.memory_allocated() / 1024**3
514
- peak = torch.cuda.max_memory_allocated() / 1024**3
515
- free, total = torch.cuda.mem_get_info()
516
- print(f"[VRAM {tag}] allocated={allocated:.2f}GB peak={peak:.2f}GB free={free / 1024**3:.2f}GB total={total / 1024**3:.2f}GB")
517
-
518
-
519
- def detect_aspect_ratio(image) -> str:
520
- if image is None:
521
- return "16:9"
522
- if hasattr(image, "size"):
523
- w, h = image.size
524
- elif hasattr(image, "shape"):
525
- h, w = image.shape[:2]
526
- else:
527
- return "16:9"
528
- ratio = w / h
529
- candidates = {"16:9": 16 / 9, "9:16": 9 / 16, "1:1": 1.0}
530
- return min(candidates, key=lambda k: abs(ratio - candidates[k]))
531
-
532
-
533
- def on_image_upload(first_image, last_image, high_res):
534
- ref_image = first_image if first_image is not None else last_image
535
- aspect = detect_aspect_ratio(ref_image)
536
- tier = "high" if high_res else "low"
537
- w, h = RESOLUTIONS[tier][aspect]
538
- return gr.update(value=w), gr.update(value=h)
539
-
540
-
541
- def on_highres_toggle(first_image, last_image, high_res):
542
- ref_image = first_image if first_image is not None else last_image
543
- aspect = detect_aspect_ratio(ref_image)
544
- tier = "high" if high_res else "low"
545
- w, h = RESOLUTIONS[tier][aspect]
546
- return gr.update(value=w), gr.update(value=h)
547
-
548
-
549
- def get_gpu_duration(
550
- first_image,
551
- last_image,
552
- prompt: str,
553
- negative_prompt: str,
554
- duration: float,
555
- gpu_duration: float,
556
- enhance_prompt: bool = True,
557
- seed: int = 42,
558
- randomize_seed: bool = True,
559
- height: int = 1024,
560
- width: int = 1536,
561
- pose_strength: float = 0.0,
562
- general_strength: float = 0.0,
563
- motion_strength: float = 0.0,
564
- dreamlay_strength: float = 0.0,
565
- mself_strength: float = 0.0,
566
- dramatic_strength: float = 0.0,
567
- fluid_strength: float = 0.0,
568
- liquid_strength: float = 0.0,
569
- demopose_strength: float = 0.0,
570
- voice_strength: float = 0.0,
571
- realism_strength: float = 0.0,
572
- transition_strength: float = 0.0,
573
- progress=None,
574
- ):
575
- return int(gpu_duration)
576
-
577
- @spaces.GPU(duration=get_gpu_duration)
578
- @torch.inference_mode()
579
- def generate_video(
580
- first_image,
581
- last_image,
582
- prompt: str,
583
- negative_prompt: str,
584
- duration: float,
585
- gpu_duration: float,
586
- enhance_prompt: bool = True,
587
- seed: int = 42,
588
- randomize_seed: bool = True,
589
- height: int = 1024,
590
- width: int = 1536,
591
- pose_strength: float = 0.0,
592
- general_strength: float = 0.0,
593
- motion_strength: float = 0.0,
594
- dreamlay_strength: float = 0.0,
595
- mself_strength: float = 0.0,
596
- dramatic_strength: float = 0.0,
597
- fluid_strength: float = 0.0,
598
- liquid_strength: float = 0.0,
599
- demopose_strength: float = 0.0,
600
- voice_strength: float = 0.0,
601
- realism_strength: float = 0.0,
602
- transition_strength: float = 0.0,
603
- progress=gr.Progress(track_tqdm=True),
604
- ):
605
- try:
606
- torch.cuda.reset_peak_memory_stats()
607
- log_memory("start")
608
-
609
- current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
610
-
611
- frame_rate = DEFAULT_FRAME_RATE
612
- num_frames = int(duration * frame_rate) + 1
613
- num_frames = ((num_frames - 1 + 7) // 8) * 8 + 1
614
-
615
- print(f"Generating: {height}x{width}, {num_frames} frames ({duration}s), seed={current_seed}")
616
-
617
- images = []
618
- output_dir = Path("outputs")
619
- output_dir.mkdir(exist_ok=True)
620
-
621
- if first_image is not None:
622
- temp_first_path = output_dir / f"temp_first_{current_seed}.jpg"
623
- if hasattr(first_image, "save"):
624
- first_image.save(temp_first_path)
625
- else:
626
- temp_first_path = Path(first_image)
627
- images.append(ImageConditioningInput(path=str(temp_first_path), frame_idx=0, strength=1.0))
628
-
629
- if last_image is not None:
630
- temp_last_path = output_dir / f"temp_last_{current_seed}.jpg"
631
- if hasattr(last_image, "save"):
632
- last_image.save(temp_last_path)
633
- else:
634
- temp_last_path = Path(last_image)
635
- images.append(ImageConditioningInput(path=str(temp_last_path), frame_idx=num_frames - 1, strength=1.0))
636
-
637
- tiling_config = TilingConfig.default()
638
- video_chunks_number = get_video_chunks_number(num_frames, tiling_config)
639
-
640
- log_memory("before pipeline call")
641
-
642
- apply_prepared_lora_config_to_pipeline()
643
-
644
- video, audio = pipeline(
645
- prompt=prompt,
646
- negative_prompt=negative_prompt,
647
- seed=current_seed,
648
- height=int(height),
649
- width=int(width),
650
- num_frames=num_frames,
651
- frame_rate=frame_rate,
652
- images=images,
653
- tiling_config=tiling_config,
654
- enhance_prompt=enhance_prompt,
655
- )
656
-
657
- log_memory("after pipeline call")
658
-
659
- output_path = tempfile.mktemp(suffix=".mp4")
660
- encode_video(
661
- video=video,
662
- fps=frame_rate,
663
- audio=audio,
664
- output_path=output_path,
665
- video_chunks_number=video_chunks_number,
666
- )
667
-
668
- log_memory("after encode_video")
669
- return str(output_path), current_seed
670
-
671
- except Exception as e:
672
- import traceback
673
- log_memory("on error")
674
- print(f"Error: {str(e)}\n{traceback.format_exc()}")
675
- return None, current_seed
676
-
677
-
678
- with gr.Blocks(title="LTX-2.3 Distilled") as demo:
679
- gr.Markdown("# LTX-2.3 F2LF with Fast Audio-Video Generation with Frame Conditioning")
680
-
681
-
682
- with gr.Row():
683
- with gr.Column():
684
- with gr.Row():
685
- first_image = gr.Image(label="First Frame (Optional)", type="pil")
686
- last_image = gr.Image(label="Last Frame (Optional)", type="pil")
687
- prompt = gr.Textbox(
688
- label="Prompt",
689
- info="for best results - make it as elaborate as possible",
690
- value="Make this image come alive with cinematic motion, smooth animation",
691
- lines=3,
692
- placeholder="Describe the motion and animation you want...",
693
- )
694
- negative_prompt = gr.Textbox(
695
- label="Negative Prompt",
696
- value="",
697
- lines=2,
698
- placeholder="Describe what you want to avoid...",
699
- )
700
- duration = gr.Slider(label="Duration (seconds)", minimum=1.0, maximum=30.0, value=10.0, step=0.1)
701
-
702
-
703
- generate_btn = gr.Button("Generate Video", variant="primary", size="lg")
704
-
705
- with gr.Accordion("Advanced Settings", open=False):
706
- seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, value=10, step=1)
707
- randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
708
- with gr.Row():
709
- width = gr.Number(label="Width", value=1536, precision=0)
710
- height = gr.Number(label="Height", value=1024, precision=0)
711
- with gr.Row():
712
- enhance_prompt = gr.Checkbox(label="Enhance Prompt", value=False)
713
- high_res = gr.Checkbox(label="High Resolution", value=True)
714
- with gr.Column():
715
- gr.Markdown("### LoRA adapter strengths (set to 0 to disable; slow and WIP)")
716
- pose_strength = gr.Slider(
717
- label="Anthro Enhancer strength",
718
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
719
- )
720
- general_strength = gr.Slider(
721
- label="Reasoning Enhancer strength",
722
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
723
- )
724
- motion_strength = gr.Slider(
725
- label="Anthro Posing Helper strength",
726
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
727
- )
728
- dreamlay_strength = gr.Slider(
729
- label="Dreamlay strength",
730
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
731
- )
732
- mself_strength = gr.Slider(
733
- label="Mself strength",
734
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
735
- )
736
- dramatic_strength = gr.Slider(
737
- label="Dramatic strength",
738
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
739
- )
740
- fluid_strength = gr.Slider(
741
- label="Fluid Helper strength",
742
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
743
- )
744
- liquid_strength = gr.Slider(
745
- label="Liquid Helper strength",
746
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
747
- )
748
- demopose_strength = gr.Slider(
749
- label="Audio Helper strength",
750
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
751
- )
752
- voice_strength = gr.Slider(
753
- label="Voice Helper strength",
754
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
755
- )
756
- realism_strength = gr.Slider(
757
- label="Anthro Realism strength",
758
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
759
- )
760
- transition_strength = gr.Slider(
761
- label="Transition strength",
762
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
763
- )
764
- prepare_lora_btn = gr.Button("Prepare / Load LoRA Cache", variant="secondary")
765
- lora_status = gr.Textbox(
766
- label="LoRA Cache Status",
767
- value="No LoRA state prepared yet.",
768
- interactive=False,
769
- )
770
-
771
- with gr.Column():
772
- output_video = gr.Video(label="Generated Video", autoplay=False)
773
- gpu_duration = gr.Slider(
774
- label="ZeroGPU duration (seconds; 10 second Img2Vid with 1024x1024 and LoRAs = ~70)",
775
- minimum=30.0,
776
- maximum=240.0,
777
- value=75.0,
778
- step=1.0,
779
- )
780
-
781
- gr.Examples(
782
- examples=[
783
- [
784
- None,
785
- "pinkknit.jpg",
786
- "The camera falls downward through darkness as if dropped into a tunnel. "
787
- "As it slows, five friends wearing pink knitted hats and sunglasses lean "
788
- "over and look down toward the camera with curious expressions. The lens "
789
- "has a strong fisheye effect, creating a circular frame around them. They "
790
- "crowd together closely, forming a symmetrical cluster while staring "
791
- "directly into the lens.",
792
- "",
793
- 3.0,
794
- 80.0,
795
- False,
796
- 42,
797
- True,
798
- 1024,
799
- 1024,
800
- 0.0, # pose_strength (example)
801
- 0.0, # general_strength (example)
802
- 0.0, # motion_strength (example)
803
- 0.0,
804
- 0.0,
805
- 0.0,
806
- 0.0,
807
- 0.0,
808
- 0.0,
809
- 0.0,
810
- 0.0,
811
- 0.0,
812
- ],
813
- ],
814
- inputs=[
815
- first_image, last_image, prompt, negative_prompt, duration, gpu_duration,
816
- enhance_prompt, seed, randomize_seed, height, width,
817
- pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength,
818
- ],
819
- )
820
-
821
- first_image.change(
822
- fn=on_image_upload,
823
- inputs=[first_image, last_image, high_res],
824
- outputs=[width, height],
825
- )
826
-
827
- last_image.change(
828
- fn=on_image_upload,
829
- inputs=[first_image, last_image, high_res],
830
- outputs=[width, height],
831
- )
832
-
833
- high_res.change(
834
- fn=on_highres_toggle,
835
- inputs=[first_image, last_image, high_res],
836
- outputs=[width, height],
837
- )
838
-
839
- prepare_lora_btn.click(
840
- fn=prepare_lora_cache,
841
- inputs=[pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength],
842
- outputs=[lora_status],
843
- )
844
-
845
- generate_btn.click(
846
- fn=generate_video,
847
- inputs=[
848
- first_image, last_image, prompt, negative_prompt, duration, gpu_duration, enhance_prompt,
849
- seed, randomize_seed, height, width,
850
- pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength,
851
- ],
852
- outputs=[output_video, seed],
853
- )
854
-
855
-
856
- css = """
857
- .fillable{max-width: 1200px !important}
858
- """
859
-
860
- if __name__ == "__main__":
861
- demo.launch(theme=gr.themes.Citrus(), css=css)