Upload 9 files

README.md

@@ -1,3 +1,32 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+---
+
+
+## Quick Start
+Please Refer to this [Repo](https://github.com/Freeze-Omni-MLLM/Freeze-Omni).
+
+## ACCEPTABLE USE POLICY
+
+Any license on the model is subject to your compliance with the Acceptable Use Policy, and You must not violate (or encourage or permit anyone else to violate) any term of the Acceptable Use Policy. Tencent reserves the right to update this Acceptable Use Policy from time to time.
+
+Tencent endeavors to promote safe and fair use of its tools and features, including Freeze-Omni. You agree not to use Freeze-Omni or any of its derivatives:
+1. In any way that violates any applicable national, federal, state, local, international or any other law or regulation;
+2. To harm Yourself or others;
+3. To repurpose or distribute output from Freeze-Omni or any of its derivatives to harm Yourself or others; 
+4. To override or circumvent the safety guardrails and safeguards We have put in place;
+5. For the purpose of exploiting, harming or attempting to exploit or harm minors in any way;
+6. To generate or disseminate verifiably false information and/or content with the purpose of harming others or influencing elections;
+7. To generate or facilitate false online engagement, including fake reviews and other means of fake online engagement;
+8. To intentionally defame, disparage or otherwise harass others;
+9. To generate and/or disseminate malware (including ransomware) or any other content to be used for the purpose of harming electronic systems;
+10. To generate or disseminate personal identifiable information with the purpose of harming others;
+11. To generate or disseminate information (including images, code, posts, articles), and place the information in any public context (including –through the use of bot generated tweets), without expressly and conspicuously identifying that the information and/or content is machine generated;
+12. To impersonate another individual without consent, authorization, or legal right;
+13. To make high-stakes automated decisions in domains that affect an individual’s safety, rights or wellbeing (e.g., law enforcement, migration, medicine/health, management of critical infrastructure, safety components of products, essential services, credit, employment, housing, education, social scoring, or insurance);
+14. In a manner that violates or disrespects the social ethics and moral standards of other countries or regions;
+15. To perform, facilitate, threaten, incite, plan, promote or encourage violent extremism or terrorism;
+16. For any use intended to discriminate against or harm individuals or groups based on protected characteristics or categories, online or offline social behavior or known or predicted personal or personality characteristics;
+17. To intentionally exploit any of the vulnerabilities of a specific group of persons based on their age, social, physical or mental characteristics, in order to materially distort the behavior of a person pertaining to that group in a manner that causes or is likely to cause that person or another person physical or psychological harm;
+18. For military purposes;
+19. To engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or other professional practices.

checkpoints/audiollm/final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d90043f43772c1c9593e818d42cf1b7a151eb6407492f6bcf940d0f0105054
+size 134

checkpoints/audiollm/global_cmvn

@@ -0,0 +1 @@
+{"mean_stat": [42290933760.0, 44562989056.0, 47934054400.0, 50294329344.0, 52492623872.0, 54071517184.0, 55109054464.0, 55449354240.0, 55966240768.0, 56030208000.0, 56560594944.0, 56809668608.0, 57317167104.0, 57670230016.0, 57685004288.0, 57780109312.0, 57635872768.0, 57397317632.0, 57772961792.0, 57156792320.0, 56690987008.0, 57452654592.0, 56838529024.0, 57277726720.0, 56979447808.0, 57289629696.0, 56877248512.0, 57206501376.0, 56933548032.0, 56858759168.0, 56982441984.0, 56856772608.0, 56879472640.0, 57088770048.0, 57116323840.0, 57327616000.0, 57559212032.0, 57580515328.0, 57627168768.0, 57469554688.0, 57644437504.0, 57513947136.0, 57888034816.0, 57862254592.0, 58113445888.0, 58480062464.0, 58896785408.0, 59059277824.0, 59257237504.0, 59599949824.0, 59462504448.0, 59389997056.0, 59443486720.0, 59678318592.0, 59872899072.0, 60087185408.0, 60333211648.0, 60344242176.0, 60080939008.0, 59774820352.0, 59574546432.0, 59173933056.0, 58974326784.0, 58689245184.0, 58367008768.0, 57964703744.0, 57601159168.0, 57429585920.0, 57327013888.0, 57154965504.0, 57004187648.0, 56897101824.0, 56751833088.0, 56619937792.0, 56493240320.0, 56417366016.0, 56087306240.0, 55201955840.0, 53744488448.0, 51637952512.0], "var_stat": [532830093312.0, 594386157568.0, 684045107200.0, 747270963200.0, 810570416128.0, 858795933696.0, 892044771328.0, 904934653952.0, 920622006272.0, 921166610432.0, 936514224128.0, 944255991808.0, 960265256960.0, 971787075584.0, 972522848256.0, 975147433984.0, 970122919936.0, 961723301888.0, 973371146240.0, 953644548096.0, 939215552512.0, 962715451392.0, 943020900352.0, 956426420224.0, 946717458432.0, 956588097536.0, 943218753536.0, 953962921984.0, 945552752640.0, 942825799680.0, 946685935616.0, 942276673536.0, 942942650368.0, 949440217088.0, 950289104896.0, 957144694784.0, 964650074112.0, 965387616256.0, 966771015680.0, 961970044928.0, 967419625472.0, 963353182208.0, 975578660864.0, 974970945536.0, 983098916864.0, 995290775552.0, 1009610391552.0, 1015431233536.0, 1022162108416.0, 1033565700096.0, 1029262868480.0, 1027019636736.0, 1028913233920.0, 1037071613952.0, 1044418330624.0, 1052328067072.0, 1060522426368.0, 1060893556736.0, 1052188213248.0, 1041587306496.0, 1035020271616.0, 1021935616000.0, 1015461838848.0, 1006602878976.0, 996370087936.0, 983805853696.0, 972873465856.0, 967858782208.0, 965317427200.0, 960507281408.0, 956465217536.0, 953962856448.0, 950124937216.0, 946753961984.0, 942988001280.0, 939914231808.0, 929753333760.0, 902898647040.0, 858106888192.0, 795279425536.0], "frame_num": 3572697669}

checkpoints/audiollm/train.yaml

@@ -0,0 +1,125 @@
+accum_grad: 1
+cmvn_file: 
+dataset_conf:
+  batch_conf:
+    batch_size: 4
+    batch_type: chat
+  fbank_conf:
+    chunk_size: 4
+    dither: 0.0
+    frame_length: 25
+    frame_shift: 10
+    num_mel_bins: 80
+    pad_rate: 0.1
+  filter_conf:
+    max_length: 2000
+    max_output_input_ratio: 102400
+    max_total_num: 1000
+    min_length: 10
+    min_output_input_ratio: 0
+    token_max_length: 102400
+    token_min_length: 0
+  parse_multi_rounds: true
+  resample_conf:
+    resample_rate: 16000
+  shuf_list: true
+  shuffle: false
+  sort: false
+  spec_aug: true
+  spec_aug_conf:
+    max_f: 10
+    max_t: 20
+    num_f_mask: 1
+    num_t_mask: 1
+  spec_sub: false
+  spec_sub_conf:
+    max_t: 30
+    num_t_sub: 3
+  speed_perturb: false
+  split_num: 1
+  tokenize_char: false
+  tokenize_conf:
+    eod_id: 151645
+  tokenize_type: Qwen
+  tokenizer_path: 
+ds_dtype: bf16
+encoder_conf:
+  overview_conf:
+    encoder-input-dim: 80
+    encoder-layer-config: subsampling-transformer
+    encoder-output-dim: 1024
+  para_conf:
+    subsampling:
+      subsampling-dropout-rate: 0.1
+      subsampling-input-dim: 80
+      subsampling-output-dim: 1024
+      subsampling-rate: 4
+    transformer:
+      transformer-attention-dim: 1024
+      transformer-attention-dropout-rate: 0.0
+      transformer-attention-heads: 16
+      transformer-chunk_size: 4
+      transformer-concat-after: false
+      transformer-dropout-rate: 0.1
+      transformer-dynamic-chunks: false
+      transformer-input-dim: 1024
+      transformer-input-layer: linear
+      transformer-left_chunks: 16
+      transformer-linear-units: 4096
+      transformer-normalize-before: true
+      transformer-num-blocks: 24
+      transformer-output-dim: 1024
+      transformer-pos-enc-class: rel-enc
+      transformer-positional-dropout-rate: 0.1
+      transformer-positionwise-layer-type: linear
+grad_clip: 5.0
+input_dim: 80
+is_json_cmvn: true
+lang_dict: 
+lfmmi_dir: ''
+log_interval: 100
+max_epoch: 100
+model_conf:
+  activation_func: gelu
+  add_audio_bos_eos: true
+  add_prompt_before: true
+  adpter_type: subsampling
+  chat_template: '<|im_start|>system
+
+    You are a helpful assistant.<|im_end|>
+
+    <|im_start|>user
+
+    <audio><|im_end|>
+
+    <|im_start|>assistant
+
+    '
+  chunk_size: 2
+  enc_out_dim: 1024
+  freeze_adpter: true
+  freeze_encoder: true
+  freeze_llm: true
+  kernel_size: 5
+  llm_embed_dim: 3584
+  llm_head_num: 28
+  llm_path: 
+  norm: layer
+  num_key_value_heads: 4
+  predict_usr_state: 4
+  prompt_finetune: true
+  prompt_num: 25
+  task_num: 20
+  task_type: prompt_finetune
+optim: adamw
+optim_conf:
+  betas:
+  - 0.9
+  - 0.99
+  eps: 1.0e-06
+  lr: 0.0006
+  weight_decay: 0.01
+output_dim: 5538
+scheduler: warmuplr
+scheduler_conf:
+  warmup_steps: 200

checkpoints/codec/final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3835f23940dee376066b65bac0fc22004605f0c7f9900da7ba3381af4c03913
+size 134

checkpoints/codec/model.json

@@ -0,0 +1,40 @@
+{
+    "resblock": "1",
+    "num_gpus": 8,
+    "batch_size": 160,
+    "learning_rate": 0.0002,
+    "adam_b1": 0.5,
+    "adam_b2": 0.9,
+    "lr_decay": 0.98,
+    "seed": 1234,
+
+    "upsample_rates": [8,5,5,3],
+    "upsample_kernel_sizes": [16,11,11,5],
+    "upsample_initial_channel": 512,
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+
+    "segment_size": 24000,
+    "num_mels": 80,
+    "num_freq": 1025,
+    "n_fft": 1024,
+    "hop_size": 240,
+    "win_size": 1024,
+
+    "sampling_rate": 24000,
+
+    "n_code_groups": 1,
+    "residul_layer": 1,
+    "n_codes": 1024,
+    "codebook_loss_lambda": 1.0,
+    "commitment_loss_lambda": 0.25,
+    "global_code_num": 8,
+    "global_feature_conv":[128, 64, 128, 3, 1],
+    "global_tokens": [473,975,419,219,565,121,550,616],
+
+    "fmin": 0,
+    "fmax": 8000,
+    "fmax_for_loss": null,
+
+    "num_workers": 12
+}

checkpoints/decoder/final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4f98e64988171c3cd85386a0f63b43a710e7a40eb154cf3b21d365b79e4ae69
+size 135

checkpoints/decoder/model.json

@@ -0,0 +1,72 @@
+[
+    896,
+    1024,
+    {
+        "accum_grad": 3,
+        "char_list": [],
+        "debugmode": 0,
+        "encoder_criterion": "ce",
+        "encoder_drop_rate": 0.1,
+        "encoder_input_dim": 896,
+        "encoder_layer_config": "transformer",
+        "encoder_output_dim": 896,
+        "encoder_pre_norm_type": "ln",
+        "encoder_upsample_rate": 9,
+        "kv_cache_prefix_finetune": 1,
+        "epochs": 100,
+        "eps": 1e-08,
+        "eps_decay": 0.8,
+        "gpu_id": null,
+        "gpu_num": 1,
+        "grad_clip": 5,
+        "grad_noise": false,
+        "idim": 896,
+        "init_lr": 0.0005,
+        "lsm_weight": 0.0,
+        "max_batch_size": 25,
+        "max_duration": 256,
+        "max_mem": 20000,
+        "mtlalpha": 0.5,
+        "n_iter_processes": 8,
+        "noam_warmup_steps": 4000,
+        "odim": 1024,
+        "opt": "noamw",
+        "rank": 0,
+        "report_interval_iters": 100,
+        "resume_trainer": false,
+        "save_interval_iters": 2000,
+        "seed": 19832,
+        "sort_duration": true,
+        "start_decay_epoch": 5,
+        "stop_learning_rate": 1e-05,
+        "sycn_batchnorm": false,
+        "tensorboard_dir": null,
+        "train_dtype": "bfloat16",
+        "transformer_attention_dim": 896,
+        "transformer_attention_dropout_rate": 0.1,
+        "transformer_attention_heads": 14,
+        "transformer_chunk_size": [
+            1
+        ],
+        "transformer_concat_after": false,
+        "transformer_dropout_rate": 0.1,
+        "transformer_dynamic_chunks": false,
+        "transformer_input_dim": 896,
+        "transformer_input_layer": "linear",
+        "transformer_left_chunks": [
+            -1
+        ],
+        "transformer_linear_units": 4864,
+        "transformer_normalize_before": true,
+        "transformer_num_blocks": 4,
+        "transformer_output_dim": 896,
+        "transformer_pos_enc_class": "rel-enc",
+        "transformer_positional_dropout_rate": 0.1,
+        "transformer_positionwise_conv_kernel_size": 1,
+        "transformer_positionwise_layer_type": "linear",
+        "use_zero_redun_opt": false,
+        "verbose": 0,
+        "weight_decay": 0.05,
+        "world_size": 1
+    }
+]

checkpoints/server.json

@@ -0,0 +1,11 @@
+{
+    "decoder_first_chunk_size": 20,
+    "decoder_chunk_size": 40,
+    "decoder_chunk_overlap_size": 10,
+    "decoder_top_k": 2,
+    "decoder_penalty_window_size": -1,
+    "decoder_penalty": 1.1,
+    "decoder_N": 2401,
+    "decoder_seg_threshold_first_pack": 0.1,
+    "decoder_seg_threshold": 0.015
+}