wandb_version: 1
task:
  name: bimanual_mv
  shape_meta:
    obs:
      scene_1:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_2:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_3:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_4:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_5:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_6:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_7:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_8:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_9:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_10:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_11:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      scene_12:
        shape:
        - 3
        - 256
        - 320
        type: rgb
      robot__actual__poses__right::panda__xyz:
        shape:
        - 3
        type: low_dim
      robot__actual__poses__right::panda__rot_6d:
        shape:
        - 6
        type: low_dim
      robot__actual__poses__left::panda__xyz:
        shape:
        - 3
        type: low_dim
      robot__actual__poses__left::panda__rot_6d:
        shape:
        - 6
        type: low_dim
      robot__actual__grippers__right::panda_hand:
        shape:
        - 1
        type: low_dim
      robot__actual__grippers__left::panda_hand:
        shape:
        - 1
        type: low_dim
      robot__desired__poses__right::panda__xyz:
        shape:
        - 3
        type: low_dim
      robot__desired__poses__right::panda__rot_6d:
        shape:
        - 6
        type: low_dim
      robot__desired__poses__left::panda__xyz:
        shape:
        - 3
        type: low_dim
      robot__desired__poses__left::panda__rot_6d:
        shape:
        - 6
        type: low_dim
      robot__desired__grippers__right::panda_hand:
        shape:
        - 1
        type: low_dim
      robot__desired__grippers__left::panda_hand:
        shape:
        - 1
        type: low_dim
    action:
      shape:
      - 20
  task_name: bimanual_video_gen
  dataset_path:
  - /store/real/liuzeyi/video_policy/data/BimanualPutSpatulaOnTableFromUtensilCrock/2025-03-29T06-34-17+00-00/diffusion_spartan/episode_*
  - /store/real/liuzeyi/video_policy/data/BimanualPutSpatulaOnTableFromUtensilCrock/2025-03-29T07-14-47+00-00/diffusion_spartan/episode_*
  dataset:
    _target_: dataset.spartan_video_dataset.SpartanVideoMultiViewDataset
    episode_path_globs:
    - /store/real/liuzeyi/video_policy/data/BimanualPutSpatulaOnTableFromUtensilCrock/2025-03-29T06-34-17+00-00/diffusion_spartan/episode_*
    - /store/real/liuzeyi/video_policy/data/BimanualPutSpatulaOnTableFromUtensilCrock/2025-03-29T07-14-47+00-00/diffusion_spartan/episode_*
    stride: 5
    horizon: 11
    pad_before: 0
    pad_after: 0
    shape_meta:
      obs:
        scene_1:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_2:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_3:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_4:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_5:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_6:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_7:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_8:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_9:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_10:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_11:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        scene_12:
          shape:
          - 3
          - 256
          - 320
          type: rgb
        robot__actual__poses__right::panda__xyz:
          shape:
          - 3
          type: low_dim
        robot__actual__poses__right::panda__rot_6d:
          shape:
          - 6
          type: low_dim
        robot__actual__poses__left::panda__xyz:
          shape:
          - 3
          type: low_dim
        robot__actual__poses__left::panda__rot_6d:
          shape:
          - 6
          type: low_dim
        robot__actual__grippers__right::panda_hand:
          shape:
          - 1
          type: low_dim
        robot__actual__grippers__left::panda_hand:
          shape:
          - 1
          type: low_dim
        robot__desired__poses__right::panda__xyz:
          shape:
          - 3
          type: low_dim
        robot__desired__poses__right::panda__rot_6d:
          shape:
          - 6
          type: low_dim
        robot__desired__poses__left::panda__xyz:
          shape:
          - 3
          type: low_dim
        robot__desired__poses__left::panda__rot_6d:
          shape:
          - 6
          type: low_dim
        robot__desired__grippers__right::panda_hand:
          shape:
          - 1
          type: low_dim
        robot__desired__grippers__left::panda_hand:
          shape:
          - 1
          type: low_dim
      action:
        shape:
        - 20
    rotation_rep: rotation_6d
    repeat_head: 0
    repeat_tail: 0
    has_gripper: false
    has_depth: true
    has_label: true
    val_ratio: 0.1
    imagenet_normalization: false
    mode: zarr
    replay_buffer_path: /local/real/liuzeyi/video_policy/data/single_task/put_spatula_on_table/replay_buffer_v2.zarr
    compressor: blosc
    is_multiarm: true
    num_workers: 16
    raw_rgb: false
    apply_static_filter: true
    vae_3d_encoder: false
name: train_video_diffusion
task_name: bimanual_mv
shape_meta:
  obs:
    scene_1:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_2:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_3:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_4:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_5:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_6:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_7:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_8:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_9:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_10:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_11:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    scene_12:
      shape:
      - 3
      - 256
      - 320
      type: rgb
    robot__actual__poses__right::panda__xyz:
      shape:
      - 3
      type: low_dim
    robot__actual__poses__right::panda__rot_6d:
      shape:
      - 6
      type: low_dim
    robot__actual__poses__left::panda__xyz:
      shape:
      - 3
      type: low_dim
    robot__actual__poses__left::panda__rot_6d:
      shape:
      - 6
      type: low_dim
    robot__actual__grippers__right::panda_hand:
      shape:
      - 1
      type: low_dim
    robot__actual__grippers__left::panda_hand:
      shape:
      - 1
      type: low_dim
    robot__desired__poses__right::panda__xyz:
      shape:
      - 3
      type: low_dim
    robot__desired__poses__right::panda__rot_6d:
      shape:
      - 6
      type: low_dim
    robot__desired__poses__left::panda__xyz:
      shape:
      - 3
      type: low_dim
    robot__desired__poses__left::panda__rot_6d:
      shape:
      - 6
      type: low_dim
    robot__desired__grippers__right::panda_hand:
      shape:
      - 1
      type: low_dim
    robot__desired__grippers__left::panda_hand:
      shape:
      - 1
      type: low_dim
  action:
    shape:
    - 20
exp_name: default
_target_: workspace.finetune_svd_lightning_workspace.FinetuneSVDLightningWorkspace
horizon: 11
n_obs_steps: 11
n_action_steps: 0
n_latency_steps: 0
past_action_visible: false
keypoint_visible_rate: 1.0
obs_as_global_cond: true
model:
  base_learning_rate: 1.0e-06
  target: sgm.models.diffusion.DiffusionEngine
  params:
    scale_factor: 0.18215
    disable_first_stage_autocast: true
    disable_loss_fn_autocast: true
    input_mv_keys:
    - pointmap
    - pointmap_right
    ckpt_path: /store/real/liuzeyi/video_policy/outputs/2025.04.02/17.13.03_train_video_diffusion_bimanual_mv/checkpoints/epoch=187-train_loss=0.015224.ckpt
    use_ema: false
    ema_decay_rate: 0.9999
    en_and_decode_n_samples_a_time: 10
    ft_strategy: everything
    optimizer_config:
      target: torch.optim.Adam
    denoiser_config:
      target: sgm.modules.diffusionmodules.denoiser.Denoiser
      params:
        scaling_config:
          target: sgm.modules.diffusionmodules.denoiser_scaling.VScalingWithEDMcNoise
    network_config:
      target: sgm.modules.diffusionmodules.video_model.VideoUNet
      params:
        adm_in_channels: 768
        num_classes: null
        use_checkpoint: true
        in_channels: 8
        out_channels: 4
        model_channels: 320
        attention_resolutions:
        - 4
        - 2
        - 1
        num_res_blocks: 2
        channel_mult:
        - 1
        - 2
        - 4
        - 4
        num_head_channels: 64
        use_linear_in_transformer: true
        transformer_depth: 1
        context_dim: 1024
        spatial_transformer_attn_type: softmax-xformers
        extra_ff_mix_layer: true
        use_spatial_context: true
        merge_strategy: learned_with_images
        video_kernel_size:
        - 3
        - 1
        - 1
        share_decoder_weights: false
        use_attn_encoder: false
        use_attn_mid: false
        use_attn_decoder: true
    conditioner_config:
      target: sgm.modules.GeneralConditioner
      params:
        emb_models:
        - input_keys:
          - cond_images_without_noise
          - cond_images_without_noise_right
          is_trainable: false
          ucg_rate: 0.1
          target: sgm.modules.encoders.modules.FrozenOpenCLIPImagePredictionEmbedder
          params:
            n_cond_frames: 1
            n_copies: 1
            open_clip_embedding_config:
              target: sgm.modules.encoders.modules.FrozenOpenCLIPImageEmbedder
              params:
                freeze: true
                num_image_crops: 10
        - input_keys:
          - cond_pointmaps_without_noise
          - cond_pointmaps_without_noise_right
          is_trainable: false
          ucg_rate: 0.1
          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
          params:
            disable_encoder_autocast: true
            en_and_decode_n_samples_a_time: 10
            n_cond_frames: 1
            n_copies: 1
            is_ae: true
            encoder_config:
              target: sgm.models.autoencoder.AutoencoderKLModeOnly
              params:
                embed_dim: 4
                ckpt_path: /store/real/liuzeyi/video_policy/outputs/2025.04.02/01.56.05_finetune_video_diffusion_autoencoder_bimanual_mv/checkpoints/epoch_90-step_27118.ckpt
                monitor: val/rec_loss
                ddconfig:
                  attn_type: vanilla-xformers
                  double_z: true
                  z_channels: 4
                  resolution: 256
                  in_channels: 3
                  out_ch: 3
                  ch: 128
                  ch_mult:
                  - 1
                  - 2
                  - 4
                  - 4
                  num_res_blocks: 2
                  attn_resolutions: []
                  dropout: 0.0
                lossconfig:
                  target: torch.nn.Identity
        - input_keys:
          - cond_images_without_noise
          - cond_images_without_noise_right
          is_trainable: false
          ucg_rate: 0.1
          target: sgm.modules.encoders.modules.VideoPredictionEmbedderWithEncoder
          params:
            disable_encoder_autocast: true
            en_and_decode_n_samples_a_time: 10
            n_cond_frames: 1
            n_copies: 1
            is_ae: true
            encoder_config:
              target: sgm.models.autoencoder.AutoencoderKLModeOnly
              params:
                embed_dim: 4
                ckpt_path: /store/real/liuzeyi/video_policy/outputs/2025.04.02/01.50.17_finetune_video_diffusion_autoencoder_bimanual_video_gen/checkpoints/epoch_40-step_11234.ckpt
                monitor: val/rec_loss
                ddconfig:
                  attn_type: vanilla-xformers
                  double_z: true
                  z_channels: 4
                  resolution: 256
                  in_channels: 3
                  out_ch: 3
                  ch: 128
                  ch_mult:
                  - 1
                  - 2
                  - 4
                  - 4
                  num_res_blocks: 2
                  attn_resolutions: []
                  dropout: 0.0
                lossconfig:
                  target: torch.nn.Identity
    sampler_config:
      target: sgm.modules.diffusionmodules.sampling.EulerEDMSampler
      params:
        num_steps: 25
        discretization_config:
          target: sgm.modules.diffusionmodules.discretizer.EDMDiscretization
          params:
            sigma_max: 700.0
        guider_config:
          target: sgm.modules.diffusionmodules.guiders.LinearPredictionGuider
          params:
            num_frames: 10
            max_scale: 2.5
            min_scale: 1.0
    loss_fn_config:
      target: sgm.modules.diffusionmodules.loss.StandardDiffusionLoss
      params:
        harmonize_sigmas: true
        batch2model_keys:
        - image_only_indicator
        - num_video_frames
        loss_weighting_config:
          target: sgm.modules.diffusionmodules.loss_weighting.EDMWeighting
          params:
            sigma_data: 1.0
        sigma_sampler_config:
          target: sgm.modules.diffusionmodules.sigma_sampling.EDMSampling
          params:
            p_mean: 1.0
            p_std: 1.6
    first_stage_pointmap_config:
      target: sgm.models.autoencoder.AutoencodingEngine
      params:
        loss_config:
          target: torch.nn.Identity
        regularizer_config:
          target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
        encoder_config:
          target: sgm.modules.diffusionmodules.model.Encoder
          params:
            attn_type: vanilla
            double_z: true
            z_channels: 4
            resolution: 256
            in_channels: 3
            out_ch: 3
            ch: 128
            ch_mult:
            - 1
            - 2
            - 4
            - 4
            num_res_blocks: 2
            attn_resolutions: []
            dropout: 0.0
        decoder_config:
          target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
          params:
            attn_type: vanilla
            double_z: true
            z_channels: 4
            resolution: 256
            in_channels: 3
            out_ch: 3
            ch: 128
            ch_mult:
            - 1
            - 2
            - 4
            - 4
            num_res_blocks: 2
            attn_resolutions: []
            dropout: 0.0
            video_kernel_size:
            - 3
            - 1
            - 1
        ckpt_path: /store/real/liuzeyi/video_policy/outputs/2025.04.02/01.56.05_finetune_video_diffusion_autoencoder_bimanual_mv/checkpoints/epoch_90-step_27118.ckpt
    first_stage_color_config:
      target: sgm.models.autoencoder.AutoencodingEngine
      params:
        loss_config:
          target: torch.nn.Identity
        regularizer_config:
          target: sgm.modules.autoencoding.regularizers.DiagonalGaussianRegularizer
        encoder_config:
          target: sgm.modules.diffusionmodules.model.Encoder
          params:
            attn_type: vanilla
            double_z: true
            z_channels: 4
            resolution: 256
            in_channels: 3
            out_ch: 3
            ch: 128
            ch_mult:
            - 1
            - 2
            - 4
            - 4
            num_res_blocks: 2
            attn_resolutions: []
            dropout: 0.0
        decoder_config:
          target: sgm.modules.autoencoding.temporal_ae.VideoDecoder
          params:
            attn_type: vanilla
            double_z: true
            z_channels: 4
            resolution: 256
            in_channels: 3
            out_ch: 3
            ch: 128
            ch_mult:
            - 1
            - 2
            - 4
            - 4
            num_res_blocks: 2
            attn_resolutions: []
            dropout: 0.0
            video_kernel_size:
            - 3
            - 1
            - 1
        ckpt_path: /store/real/liuzeyi/video_policy/outputs/2025.04.02/01.50.17_finetune_video_diffusion_autoencoder_bimanual_video_gen/checkpoints/epoch_40-step_11234.ckpt
dataloader:
  batch_size: 1
  num_workers: 16
  shuffle: true
  pin_memory: true
  persistent_workers: true
val_dataloader:
  batch_size: 1
  num_workers: 16
  shuffle: false
  pin_memory: true
  persistent_workers: true
optimizer:
  _target_: torch.optim.AdamW
  lr: 0.0001
  betas:
  - 0.95
  - 0.999
  eps: 1.0e-08
  weight_decay: 1.0e-06
training:
  output_dir: /store/real/liuzeyi/video_policy/outputs/2025.04.02/17.13.03_train_video_diffusion_bimanual_mv
  device: -1
  num_nodes: 1
  seed: 42
  debug: false
  lr_scheduler: cosine
  lr_warmup_steps: 50
  lr_step_interval: epoch
  num_epochs: 1000
  gradient_accumulate_every: 4
  gradient_clip_val: 1.0
  gradient_clip_algorithm: norm
  use_ema: false
  precision: 16
  strategy: ddp
  freeze_encoder: false
  train_with_features: false
  rollout_every: 50
  checkpoint_every: 50
  periodic_save: 100
  val_every: 1000
  val_per_step: true
  sample_every: 5
  max_train_steps: null
  max_val_steps: 1
  tqdm_interval_sec: 1.0
logging:
  project: video-gen
  mode: offline
  name: 2025.04.02-17.13.03_train_video_diffusion_bimanual_mv
  tags:
  - train_video_diffusion
  - bimanual_mv
  - default
  - pl
  - batch_scaling
checkpoint:
  topk:
    monitor_key: train_loss
    mode: min
    k: 3
    format_str: '{epoch:03d}-{train_loss:.6f}'
  save_last_ckpt: false
  save_last_snapshot: false
  every_n_epoch: 10
  log_prediction_sample: true
_wandb:
  python_version: 3.10.12
  cli_version: 0.17.2
  framework: huggingface
  huggingface_version: 4.21.1
  is_jupyter_run: false
  is_kaggle_kernel: false
  start_time: 1743613987
  t:
    1:
    - 1
    - 9
    - 11
    - 41
    - 49
    - 50
    - 55
    - 63
    - 71
    - 79
    - 103
    2:
    - 1
    - 9
    - 11
    - 30
    - 41
    - 49
    - 50
    - 55
    - 63
    - 71
    - 79
    - 103
    3:
    - 3
    - 13
    - 15
    - 16
    - 23
    4: 3.10.12
    5: 0.17.2
    6: 4.21.1
    8:
    - 5
    13: linux-x86_64
output_dir: /store/real/liuzeyi/video_policy/outputs/2025.04.02/17.13.03_train_video_diffusion_bimanual_mv

