first ckpt

config.yaml

@@ -0,0 +1,102 @@
+pretrained_model_path: stabilityai/stable-video-diffusion-img2vid
+output_dir: ./output/svd
+train_data:
+  width: 128
+  height: 128
+  use_bucketing: false
+  return_mask: true
+  return_motion: true
+  sample_start_idx: 1
+  fps: 7
+  frame_step: 1
+  n_sample_frames: 7
+  single_video_path: data/0X1A0A263B22CCD966.avi
+  single_video_prompt: an ultrasound video of heart
+  fallback_prompt: ''
+  path: /data/vision/billf/scratch/tianyuan/dataset/ultrasound/EchoNet-Dynamic/Videos
+  json_path: /webvid/animation1.json
+  image_dir: /vlp/datasets/images/coco
+  image_json: /vlp/datasets/images/coco/coco_karpathy_train.json
+  video_dir: /webvid/webvid/data/videos
+  video_json: /webvid/webvid/data/40K.json
+  single_img_prompt: ''
+validation_data:
+  prompt: ''
+  prompt_image: data/cond_frame0.jpg
+  sample_preview: true
+  num_frames: 14
+  width: 128
+  height: 128
+  num_inference_steps: 25
+  guidance_scale: 9
+  fps: 7
+  motion_bucket_id: 127
+  decode_chunk_size: 7
+extra_train_data: []
+dataset_types:
+- folder
+shuffle: true
+validation_steps: 9000000000.0
+trainable_modules:
+- all
+- attn1
+- attn2
+- conv_in
+- temp_conv
+- motion
+extra_unet_params: null
+extra_text_encoder_params: null
+train_batch_size: 4
+max_train_steps: 10000
+learning_rate: 5.0e-06
+scale_lr: false
+lr_scheduler: constant
+lr_warmup_steps: 0
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_weight_decay: 0
+adam_epsilon: 1.0e-08
+max_grad_norm: 1.0
+gradient_accumulation_steps: 1
+gradient_checkpointing: false
+text_encoder_gradient_checkpointing: false
+checkpointing_steps: 2500
+resume_from_checkpoint: null
+resume_step: null
+mixed_precision: fp16
+use_8bit_adam: false
+enable_xformers_memory_efficient_attention: false
+enable_torch_2_attn: true
+seed: 6
+use_offset_noise: false
+rescale_schedule: false
+offset_noise_strength: 0.1
+extend_dataset: false
+cache_latents: false
+cached_latent_dir: null
+save_pretrained_model: true
+logger_type: tensorboard
+motion_mask: false
+kwargs:
+  motion_strength: false
+  train_text_encoder: false
+  lora_version: cloneofsimo
+  use_unet_lora: false
+  use_text_lora: false
+  lora_unet_dropout: 0.1
+  lora_text_dropout: 0.1
+  save_lora_for_webui: true
+  only_lora_for_webui: false
+  unet_lora_modules:
+  - UNet3DConditionModel
+  text_encoder_lora_modules:
+  - CLIPEncoderLayer
+  lora_rank: 16
+  trainable_text_modules: null
+  dataset:
+    data_path: /data/vision/billf/scratch/tianyuan/dataset/ultrasound/EchoNet-Dynamic
+    deactivate_cache: false
+    fps: 8
+    duration: 2.0
+    grayscale: false
+    image_size: 128

image_encoder/config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "/data/vision/billf/scratch/brandonf/.cache/huggingface/hub/models--stabilityai--stable-video-diffusion-img2vid/snapshots/1171a4af7a4e620fb2359c8cb3b312c30c655b93/image_encoder",
+  "architectures": [
+    "CLIPVisionModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "image_size": 224,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 32,
+  "patch_size": 14,
+  "projection_dim": 1024,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.0"
+}

image_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed1e5af7b4042ca30ec29999a4a5cfcac90b7fb610fd05ace834f2dcbb763eab
+size 2528371296

model_index.json

@@ -0,0 +1,25 @@
+{
+  "_class_name": "StableVideoDiffusionPipeline",
+  "_diffusers_version": "0.26.0.dev0",
+  "_name_or_path": "stabilityai/stable-video-diffusion-img2vid",
+  "feature_extractor": [
+    "transformers",
+    "CLIPImageProcessor"
+  ],
+  "image_encoder": [
+    "transformers",
+    "CLIPVisionModelWithProjection"
+  ],
+  "scheduler": [
+    "diffusers",
+    "EulerDiscreteScheduler"
+  ],
+  "unet": [
+    "diffusers",
+    "UNetSpatioTemporalConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKLTemporalDecoder"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,21 @@
+{
+  "_class_name": "EulerDiscreteScheduler",
+  "_diffusers_version": "0.26.0.dev0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "interpolation_type": "linear",
+  "num_train_timesteps": 1000,
+  "prediction_type": "v_prediction",
+  "rescale_betas_zero_snr": false,
+  "set_alpha_to_one": false,
+  "sigma_max": 700.0,
+  "sigma_min": 0.002,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "timestep_spacing": "leading",
+  "timestep_type": "continuous",
+  "trained_betas": null,
+  "use_karras_sigmas": true
+}

unet/config.json

@@ -0,0 +1,38 @@
+{
+  "_class_name": "UNetSpatioTemporalConditionModel",
+  "_diffusers_version": "0.26.0.dev0",
+  "_name_or_path": "/data/vision/billf/scratch/brandonf/.cache/huggingface/hub/models--stabilityai--stable-video-diffusion-img2vid/snapshots/1171a4af7a4e620fb2359c8cb3b312c30c655b93/unet",
+  "addition_time_embed_dim": 256,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "DownBlockSpatioTemporal"
+  ],
+  "in_channels": 8,
+  "layers_per_block": 2,
+  "num_attention_heads": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "num_frames": 14,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": 768,
+  "sample_size": 96,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal"
+  ]
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e3771ba35a9d98835fe9bbadf09174797b56b67d84e037ca0a798fc02f01c4
+size 6098682464

vae/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "AutoencoderKLTemporalDecoder",
+  "_diffusers_version": "0.26.0.dev0",
+  "_name_or_path": "/data/vision/billf/scratch/brandonf/.cache/huggingface/hub/models--stabilityai--stable-video-diffusion-img2vid/snapshots/1171a4af7a4e620fb2359c8cb3b312c30c655b93/vae",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9975042d7bee021bd53a72b1af14c8627d624f6547ec9abe661b68b962b88c49
+size 391017740