Upload 12 files

latent_motion_tokenizer_trained_on_calvin/config.yaml

@@ -0,0 +1,59 @@
+_target_: latent_motion_tokenizer.src.models.latent_motion_tokenizer.LatentMotionTokenizer
+codebook_dim: 32
+commit_loss_w: 1.0
+recon_loss_w: 1.0
+perceptual_loss_w: 1.0
+
+image_encoder:
+  _target_: transformers.ViTMAEModel.from_pretrained
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+
+m_former:
+  _target_: latent_motion_tokenizer.src.models.m_former.MFormer
+  add_pooling_layer: false
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    input_hidden_size: 1024
+    num_patches: 197 # include the [CLS] token
+    attention_probs_dropout_prob: 0.0
+    hidden_act: "gelu"
+    hidden_dropout_prob: 0.0
+    hidden_size: 768 # the hidden size of MAE decoder is 512
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1e-12
+    model_type: "vit"
+    num_attention_heads: 12
+    num_hidden_layers: 4
+    qkv_bias: true
+
+vector_quantizer:
+  _target_: latent_motion_tokenizer.src.models.vector_quantizer.VectorQuantizer2
+  n_e: 128
+  e_dim: 32
+  beta: 0.25
+  remap: null
+  sane_index_shape: true
+
+decoder:
+  _target_: latent_motion_tokenizer.src.models.latent_motion_decoder.LatentMotionDecoder
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    attention_probs_dropout_prob: 0.0
+    hidden_act: "gelu"
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    image_size: 224
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1e-12
+    model_type: "vit"
+    num_attention_heads: 12
+    num_channels: 3
+    num_hidden_layers: 12
+    patch_size: 16
+    qkv_bias: true
+    encoder_stride: 16
+    num_patches: 196

latent_motion_tokenizer_trained_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b7ed635296ad798bef76381b46467f17d784e4041df5c8dbe23f152aed16dbd
+size 484939570

latent_motion_tokenizer_trained_on_oxe/config.yaml

@@ -0,0 +1,59 @@
+_target_: latent_motion_tokenizer.src.models.latent_motion_tokenizer.LatentMotionTokenizer
+codebook_dim: 32
+commit_loss_w: 1.0
+recon_loss_w: 1.0
+perceptual_loss_w: 1.0
+
+image_encoder:
+  _target_: transformers.ViTMAEModel.from_pretrained
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+
+m_former:
+  _target_: latent_motion_tokenizer.src.models.m_former.MFormer
+  add_pooling_layer: false
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    input_hidden_size: 1024
+    num_patches: 197 # include the [CLS] token
+    attention_probs_dropout_prob: 0.0
+    hidden_act: "gelu"
+    hidden_dropout_prob: 0.0
+    hidden_size: 768 # the hidden size of MAE decoder is 512
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1e-12
+    model_type: "vit"
+    num_attention_heads: 12
+    num_hidden_layers: 4
+    qkv_bias: true
+
+vector_quantizer:
+  _target_: latent_motion_tokenizer.src.models.vector_quantizer.VectorQuantizer2
+  n_e: 128
+  e_dim: 32
+  beta: 0.25
+  remap: null
+  sane_index_shape: true
+
+decoder:
+  _target_: latent_motion_tokenizer.src.models.latent_motion_decoder.LatentMotionDecoder
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    attention_probs_dropout_prob: 0.0
+    hidden_act: "gelu"
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    image_size: 224
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1e-12
+    model_type: "vit"
+    num_attention_heads: 12
+    num_channels: 3
+    num_hidden_layers: 12
+    patch_size: 16
+    qkv_bias: true
+    encoder_stride: 16
+    num_patches: 196

latent_motion_tokenizer_trained_on_oxe/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1962641b1fbf77f1d3a563f362b2206a9f07ed24e8f6fa09d55d5500c0f0245b
+size 484939570

moto_gpt_finetuned_on_calvin/config.yaml

@@ -0,0 +1,33 @@
+_target_: moto_gpt.src.models.moto_gpt.MotoGPT
+model_lang: 
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: "t5-base"
+model_vision: 
+  _target_: moto_gpt.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+model_causal_transformer:
+  _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Model
+  config: 
+    _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: "relu"
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 5
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: true
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true

moto_gpt_finetuned_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b34cb739c2db5163942afb5251c30210b9ff3cca1e47154cb763df69f37bd16
+size 364148174

moto_gpt_finetuned_on_rt1/config.yaml

@@ -0,0 +1,33 @@
+_target_: moto_gpt.src.models.moto_gpt.MotoGPT
+model_lang: 
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: "t5-base"
+model_vision: 
+  _target_: moto_gpt.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+model_causal_transformer:
+  _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Model
+  config: 
+    _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: "relu"
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 3
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: true
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true

moto_gpt_finetuned_on_rt1/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f969860566c2748b470febe89c90a2354a57aa1b0b54315bf75c07b90e1defb7
+size 364135886

moto_gpt_pretrained_on_calvin/config.yaml

@@ -0,0 +1,33 @@
+_target_: moto_gpt.src.models.moto_gpt.MotoGPT
+model_lang: 
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: "t5-base"
+model_vision: 
+  _target_: moto_gpt.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+model_causal_transformer:
+  _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Model
+  config: 
+    _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: "relu"
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 5
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: false
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true

moto_gpt_pretrained_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b74fae7c2791171560a005b6f9fc292ba8bccacebfd37ad28e000b990df3f351
+size 364113734

moto_gpt_pretrained_on_oxe/config.yaml

@@ -0,0 +1,33 @@
+_target_: moto_gpt.src.models.moto_gpt.MotoGPT
+model_lang: 
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: "t5-base"
+model_vision: 
+  _target_: moto_gpt.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: "facebook/vit-mae-large"
+model_causal_transformer:
+  _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Model
+  config: 
+    _target_: moto_gpt.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: "relu"
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 3
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: false
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true

moto_gpt_pretrained_on_oxe/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13076acf0490772a8c1bc681cb19f6c58b3434a88a2b849da16c3f196462c02c
+size 364113734