Upload all files from current directory

README.md

@@ -1,3 +1,60 @@
 ---
 license: apache-2.0
 ---
+
+# **Introduction**
+
+**`XY-Tokenizer`** is a speech codec that simultaneously models both semantic and acoustic aspects of speech, converting audio into discrete tokens and decoding them back to high-quality audio. It achieves efficient speech representation at only 1kbps with RVQ8 quantization at 12.5Hz frame rate. 
+
+-   **Paper:** [Read on arXiv](https://arxiv.org/abs/2506.23325)
+-   **Source Code:** 
+    - [GitHub Repo](https://github.com/OpenMOSS/MOSS-TTSD/tree/main/XY_Tokenizer)
+    - [Hugging Face Repo](https://huggingface.co/spaces/fnlp/MOSS-TTSD/tree/main/XY_Tokenizer)
+
+## 📚 Related Project: **[MOSS-TTSD](https://huggingface.co/fnlp/MOSS-TTSD-v0.5)**
+
+**`XY-Tokenizer`** serves as the underlying neural codec for **`MOSS-TTSD`**, our 1.7B Audio Language Model. \
+Explore **`MOSS-TTSD`** for advanced text-to-speech and other audio generation tasks on [GitHub](https://github.com/OpenMOSS/MOSS-TTSD), [Blog](http://www.open-moss.com/en/moss-ttsd/), [博客](https://www.open-moss.com/cn/moss-ttsd/), and [Space Demo](https://huggingface.co/spaces/fnlp/MOSS-TTSD).
+
+## ✨ Features
+
+- **Dual-channel modeling**: Simultaneously captures semantic meaning and acoustic details
+- **Efficient representation**: 1kbps bitrate with RVQ8 quantization at 12.5Hz
+- **High-quality audio tokenization**: Convert speech to discrete tokens and back with minimal quality loss
+- **Long audio support**: Process audio files longer than 30 seconds using chunking with overlap
+- **Batch processing**: Efficiently process multiple audio files in batches
+- **24kHz output**: Generate high-quality 24kHz audio output
+
+## 💻 Quick Start
+
+Here's how to use **`XY-Tokenizer`** with `transformers` to encode an audio file into discrete tokens and decode it back into a waveform.
+
+```python
+import torchaudio
+from transformers import AutoFeatureExtractor, AutoModel
+
+# 1. Load the feature extractor and the codec model
+model_id = "fnlp/XY-Tokenizer-TTSD-V0-hf"
+feature_extractor = AutoFeatureExtractor.from_pretrained(model_id, trust_remote_code=True)
+codec = AutoModel.from_pretrained(model_id, trust_remote_code=True).eval().to("cuda")
+
+# 2. Load and preprocess the audio
+# The model expects a 16kHz sample rate.
+wav_form, sampling_rate = torchaudio.load("examples/m1.wav")
+if sampling_rate != 16000:
+    wav_form = torchaudio.functional.resample(wav_form, orig_freq=sampling_rate, new_freq=16000)
+    
+# 3. Encode the audio into discrete codes
+input_features = feature_extractor(wav_form, sampling_rate=16000, return_attention_mask=True, return_tensors="pt")
+# The 'code' dictionary contains the discrete audio codes
+code = codec.encode(input_features)
+print(code)
+
+# 4. Decode the codes back to an audio waveform
+# The output is high-quality 24kHz audio.
+output_wav = codec.decode(code["audio_codes"], overlap_seconds=10)
+
+# 5. Save the reconstructed audio
+for i, audio in enumerate(output_wav["audio_values"]):
+    torchaudio.save(f"audio_{i}.wav", audio.cpu(), 24000)
+```

config.json

@@ -0,0 +1,120 @@
+{
+  "model_type": "xy_tokenizer",
+  "input_sample_rate": 16000,
+  "output_sample_rate": 24000,
+  "encoder_downsample_rate": 1280,
+  "decoder_upsample_rate": 1920,
+  "code_dim": 3072,
+  "params": {
+    "feature_extractor_kwargs": {
+      "chunk_length": 30,
+      "feature_size": 80,
+      "hop_length": 160,
+      "n_fft": 400,
+      "n_samples": 480000,
+      "nb_max_frames": 3000,
+      "padding_side": "right",
+      "padding_value": 0.0,
+      "sampling_rate": 16000,
+      "return_attention_mask": true,
+      "return_tensors": "pt"
+    },
+    "semantic_encoder_kwargs": {
+      "num_mel_bins": 80,
+      "sampling_rate": 16000,
+      "hop_length": 160,
+      "stride_size": 2,
+      "kernel_size": 3,
+      "d_model": 768,
+      "scale_embedding": false,
+      "max_audio_seconds": 30,
+      "encoder_layers": 12,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072,
+      "activation_function": "gelu"
+    },
+    "semantic_encoder_adapter_kwargs": {
+      "input_dim": 768,
+      "output_dim": 768,
+      "d_model": 768,
+      "max_source_positions": 1500,
+      "encoder_layers": 4,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072
+    },
+    "acoustic_encoder_kwargs": {
+      "num_mel_bins": 80,
+      "sampling_rate": 16000,
+      "hop_length": 160,
+      "stride_size": 2,
+      "kernel_size": 3,
+      "d_model": 768,
+      "scale_embedding": false,
+      "max_audio_seconds": 30,
+      "encoder_layers": 12,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072,
+      "activation_function": "gelu"
+    },
+    "pre_rvq_adapter_kwargs": {
+      "input_dim": 1536,
+      "output_dim": 768,
+      "d_model": 768,
+      "max_source_positions": 1500,
+      "encoder_layers": 4,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072
+    },
+    "downsample_kwargs": {
+      "d_model": 768,
+      "avg_pooler": 4
+    },
+    "quantizer_kwargs": {
+      "input_dim": 3072,
+      "rvq_dim": 512,
+      "output_dim": 3072,
+      "num_quantizers": 8,
+      "codebook_size": 1024,
+      "codebook_dim": 512,
+      "quantizer_dropout": 0.0
+    },
+    "post_rvq_adapter_kwargs": {
+      "input_dim": 3072,
+      "output_dim": 3072,
+      "d_model": 768,
+      "max_source_positions": 375,
+      "encoder_layers": 4,
+      "encoder_attention_heads": 12,
+      "encoder_ffn_dim": 3072
+    },
+    "upsample_kwargs": {
+      "d_model": 768,
+      "stride": 4
+    },
+    "acoustic_decoder_kwargs": {
+      "num_mel_bins": 80,
+      "sampling_rate": 16000,
+      "hop_length": 160,
+      "stride_size": 2,
+      "kernel_size": 3,
+      "d_model": 768,
+      "scale_embedding": false,
+      "max_audio_seconds": 30,
+      "decoder_layers": 12,
+      "decoder_attention_heads": 12,
+      "decoder_ffn_dim": 3072,
+      "activation_function": "gelu"
+    },
+    "vocos_kwargs": {
+      "input_channels": 80,
+      "dim": 512,
+      "intermediate_dim": 4096,
+      "num_layers": 30,
+      "n_fft": 960,
+      "hop_size": 240,
+      "padding": "same"
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.0"
+}

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "chunk_length": 30,
+  "feature_size": 80,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "sampling_rate": 16000,
+  "return_attention_mask": true,
+  "return_tensors": "pt"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fafbaf4ba0e6095be842230c4bd16ecf6d193b250718a5775f1ac7aa528d9110
+size 2137279502