Upload folder using huggingface_hub

README.md

@@ -0,0 +1,49 @@
+---
+base_model:
+- m-a-p/MERT-v0-public
+- facebook/hubert-base-ls960
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# merged_model
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [m-a-p/MERT-v0-public](https://huggingface.co/m-a-p/MERT-v0-public)
+* [facebook/hubert-base-ls960](https://huggingface.co/facebook/hubert-base-ls960)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+slices:
+  - sources:
+    - model: facebook/hubert-base-ls960
+      layer_range: [0, 12]
+    - model: m-a-p/MERT-v0-public
+      layer_range: [0, 12]
+      trast_remote_code: true
+merge_method: slerp
+base_model:
+  model: facebook/hubert-base-ls960
+  override_architecture: HubertModel
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+```

config.json

@@ -0,0 +1,75 @@
+{
+  "_name_or_path": "facebook/hubert-base-ls960",
+  "activation_dropout": 0.1,
+  "apply_spec_augment": true,
+  "architectures": [
+    "HubertModel"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.1,
+  "feat_proj_layer_norm": true,
+  "final_dropout": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "model_type": "hubert",
+  "num_attention_heads": 12,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "transformers_version": "4.37.1",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32
+}

mergekit_config.yml

@@ -0,0 +1,18 @@
+slices:
+  - sources:
+    - model: facebook/hubert-base-ls960
+      layer_range: [0, 12]
+    - model: m-a-p/MERT-v0-public
+      layer_range: [0, 12]
+      trast_remote_code: true
+merge_method: slerp
+base_model:
+  model: facebook/hubert-base-ls960
+  override_architecture: HubertModel
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5

model-00001-of-00001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f385f64327eb629a567738a5b7c44b905da97ddea246173497be346290e122fb
+size 341819976

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.2", "total_size": 341797888}, "weight_map": {"encoder.layers.0.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.0.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.0.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.1.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.1.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.10.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.10.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.11.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.11.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.2.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.2.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.3.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.3.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.4.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.4.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.5.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.5.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.6.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.6.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.7.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.7.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.8.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.8.layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.k_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.k_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.out_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.out_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.q_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.q_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.v_proj.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.attention.v_proj.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.feed_forward.intermediate_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.feed_forward.intermediate_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.feed_forward.output_dense.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.feed_forward.output_dense.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.final_layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.final_layer_norm.weight": "model-00001-of-00001.safetensors", "encoder.layers.9.layer_norm.bias": "model-00001-of-00001.safetensors", "encoder.layers.9.layer_norm.weight": "model-00001-of-00001.safetensors", "feature_projection.layer_norm.bias": "model-00001-of-00001.safetensors", "feature_projection.layer_norm.weight": "model-00001-of-00001.safetensors", "feature_projection.projection.bias": "model-00001-of-00001.safetensors", "feature_projection.projection.weight": "model-00001-of-00001.safetensors"}}