Upload speechbrain IC model

.gitattributes

@@ -25,3 +25,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/asr.ckpt filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/lm.ckpt filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/model.ckpt filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/optimizer.ckpt filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/model.ckpt filter=lfs diff=lfs merge=lfs -text
+fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/optimizer.ckpt filter=lfs diff=lfs merge=lfs -text
+pretrained_models/EncoderDecoderASR--5348169877143464308/asr.ckpt filter=lfs diff=lfs merge=lfs -text
+pretrained_models/EncoderDecoderASR--5348169877143464308/lm.ckpt filter=lfs diff=lfs merge=lfs -text
+pretrained_models/EndToEndSLU-7990244956535603082/model.ckpt filter=lfs diff=lfs merge=lfs -text

fluent-speech-commands/README.md

@@ -0,0 +1,58 @@
+# SLU recipes for Fluent Speech Commands
+This folder contains recipes for spoken language understanding (SLU) with [Fluent Speech Commands](fluent.ai/research/fluent-speech-commands/).
+
+### Tokenizer recipe
+(You don't need to run this because the other recipes download a tokenizer, but you can run this if you want to train a new tokenizer for Fluent Speech Commands.)
+
+Run this to train the tokenizer:
+
+```
+cd Tokenizer
+python train.py hparams/tokenizer_bpe51.yaml
+```
+
+### Direct recipe
+The "direct" recipe maps the input speech to directly to semantics using a seq2seq model.
+The encoder is pre-trained using the LibriSpeech seq2seq recipe.
+
+```
+cd direct
+python train.py hparams/train.yaml
+```
+
+# Results
+
+| Release | hyperparams file | Test Acc | Model link | GPUs |
+|:-------------:|:---------------------------:| -----:| -----:| --------:|
+| 21-06-03 | train.yaml | 99.60% | https://drive.google.com/drive/folders/13t2PYdedrPQoNYo_QSf6s04WXu2_vAb-?usp=sharing | 1xV100 32GB |
+
+
+# PreTrained Model + Easy-Inference
+You can find the pre-trained model with an easy-inference function on [HuggingFace](https://huggingface.co/speechbrain/slu-direct-fluent-speech-commands-librispeech-asr).
+
+
+# Training Time
+About 15 minutes for each epoch with a TESLA V100.
+
+
+# **About SpeechBrain**
+- Website: https://speechbrain.github.io/
+- Code: https://github.com/speechbrain/speechbrain/
+- HuggingFace: https://huggingface.co/speechbrain/
+
+
+# **Citing SpeechBrain**
+Please, cite SpeechBrain if you use it for your research or business.
+
+```bibtex
+@misc{speechbrain,
+  title={{SpeechBrain}: A General-Purpose Speech Toolkit},
+  author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio},
+  year={2021},
+  eprint={2106.04624},
+  archivePrefix={arXiv},
+  primaryClass={eess.AS},
+  note={arXiv:2106.04624}
+}
+```
+

fluent-speech-commands/Tokenizer/hparams/tokenizer_bpe51.yaml

@@ -0,0 +1,32 @@
+# ############################################################################
+# Tokenizer: subword BPE with unigram 51
+# Training: Fluent Speech Commands
+# Authors:  Abdel Heba 2021
+# ############################################################################
+
+output_folder: !ref results/tokenizer_bpe51/
+train_log: !ref <output_folder>/train_log.txt
+
+# Data files
+data_folder: !PLACEHOLDER # e,g. /localscratch/fluent_speech_commands_dataset
+train_csv: !ref <output_folder>/train.csv
+valid_csv: !ref <output_folder>/valid.csv
+skip_prep: False
+
+# Training parameters
+token_type: unigram  # ["unigram", "bpe", "char"]
+token_output: 51  # index(blank/eos/bos/unk) = 0
+character_coverage: 1.0
+num_sequences: 10000
+csv_read: semantics
+
+
+tokenizer: !name:speechbrain.tokenizers.SentencePiece.SentencePiece
+   model_dir: !ref <output_folder>
+   vocab_size: !ref <token_output>
+   annotation_train: !ref <train_csv>
+   annotation_read: !ref <csv_read>
+   model_type: !ref <token_type> # ["unigram", "bpe", "char"]
+   character_coverage: !ref <character_coverage>
+   num_sequences: !ref <num_sequences>
+   annotation_list_to_check: [!ref <train_csv>, !ref <valid_csv>]

fluent-speech-commands/Tokenizer/prepare.py

@@ -0,0 +1 @@
+../prepare.py

fluent-speech-commands/Tokenizer/train.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env/python3
+"""Recipe for training a BPE tokenizer for Fluent Speech Commands.
+The tokenizer coverts semantics into sub-word units that can
+be used to train a language (LM) or an acoustic model (AM).
+
+To run this recipe, do the following:
+> python train.py hparams/tokenizer_bpe51.yaml
+
+
+Authors
+ * Abdel Heba 2021
+ * Mirco Ravanelli 2021
+ * Loren Lugosch 2021
+"""
+
+import sys
+import speechbrain as sb
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.utils.distributed import run_on_main
+
+if __name__ == "__main__":
+
+    # CLI:
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If distributed_launch=True then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # 1.  # Dataset prep
+    from prepare import prepare_FSC  # noqa
+
+    # multi-gpu (ddp) save data preparation
+    run_on_main(
+        prepare_FSC,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["output_folder"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Train tokenizer
+    hparams["tokenizer"]()

fluent-speech-commands/direct/hparams/train.yaml

@@ -0,0 +1,204 @@
+# ############################################################################
+# Model: Direct SLU
+# Encoder: Pre-trained ASR encoder -> LSTM
+# Decoder: GRU + beamsearch
+# Tokens: BPE with unigram
+# losses: NLL
+# Training: Fluent Speech Commands
+# Authors:  Loren Lugosch, Mirco Ravanelli 2020
+# ############################################################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 112011
+__set_seed: !apply:torch.manual_seed [!ref <seed>]
+output_folder: !ref results/BPE51/<seed>
+save_folder: !ref <output_folder>/save
+train_log: !ref <output_folder>/train_log.txt
+
+# Data files
+data_folder: !PLACEHOLDER # e.g, /localscratch/fluent_speech_commands_dataset
+rir_folder: !ref <data_folder> # Change it if needed
+csv_train: !ref <output_folder>/train.csv
+csv_valid: !ref <output_folder>/valid.csv
+csv_test: !ref <output_folder>/test.csv
+tokenizer_file: https://www.dropbox.com/s/hvf2huofnq0sjbn/51_unigram.model?dl=1
+skip_prep: False
+# Training parameters
+number_of_epochs: 4
+batch_size: 8
+lr: 0.0003
+token_type: unigram # ["unigram", "bpe", "char"]
+sorting: random
+
+# Model parameters
+sample_rate: 16000
+emb_size: 128
+dec_neurons: 512
+output_neurons: 51 # index(eos/bos) = 0
+ASR_encoder_dim: 512
+encoder_dim: 256
+
+# Decoding parameters
+bos_index: 0
+eos_index: 0
+min_decode_ratio: 0.0
+max_decode_ratio: 10.0
+slu_beam_size: 80
+eos_threshold: 1.5
+temperature: 1.25
+
+dataloader_opts:
+    batch_size: !ref <batch_size>
+    shuffle: True
+
+epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
+    limit: !ref <number_of_epochs>
+
+# Models
+asr_model: !apply:speechbrain.pretrained.EncoderDecoderASR.from_hparams
+    source: speechbrain/asr-crdnn-rnnlm-librispeech
+    run_opts: {"device":"cuda:0"}
+
+slu_enc: !new:speechbrain.nnet.containers.Sequential
+    input_shape: [null, null, !ref <ASR_encoder_dim>]
+    lstm: !new:speechbrain.nnet.RNN.LSTM
+        input_size: !ref <ASR_encoder_dim>
+        bidirectional: True
+        hidden_size: !ref <encoder_dim>
+        num_layers: 2
+    linear: !new:speechbrain.nnet.linear.Linear
+        input_size: !ref <encoder_dim> * 2
+        n_neurons: !ref <encoder_dim>
+
+output_emb: !new:speechbrain.nnet.embedding.Embedding
+    num_embeddings: !ref <output_neurons>
+    embedding_dim: !ref <emb_size>
+
+dec: !new:speechbrain.nnet.RNN.AttentionalRNNDecoder
+    enc_dim: !ref <encoder_dim>
+    input_size: !ref <emb_size>
+    rnn_type: gru
+    attn_type: keyvalue
+    hidden_size: !ref <dec_neurons>
+    attn_dim: 512
+    num_layers: 3
+    scaling: 1.0
+    dropout: 0.0
+
+seq_lin: !new:speechbrain.nnet.linear.Linear
+    input_size: !ref <dec_neurons>
+    n_neurons: !ref <output_neurons>
+
+augment_wavedrop: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+    sample_rate: !ref <sample_rate>
+    speeds: [100]
+
+augment_speed: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+    sample_rate: !ref <sample_rate>
+    speeds: [95, 100, 105]
+
+add_rev: !new:speechbrain.lobes.augment.EnvCorrupt
+    openrir_folder: !ref <rir_folder>
+    openrir_max_noise_len: 3.0  # seconds
+    reverb_prob: 1.0
+    noise_prob: 0.0
+    noise_snr_low: 0
+    noise_snr_high: 15
+    rir_scale_factor: 1.0
+
+add_noise: !new:speechbrain.lobes.augment.EnvCorrupt
+    openrir_folder: !ref <rir_folder>
+    openrir_max_noise_len: 3.0  # seconds
+    reverb_prob: 0.0
+    noise_prob: 1.0
+    noise_snr_low: 0
+    noise_snr_high: 15
+    rir_scale_factor: 1.0
+
+add_rev_noise: !new:speechbrain.lobes.augment.EnvCorrupt
+    openrir_folder: !ref <rir_folder>
+    openrir_max_noise_len: 3.0  # seconds
+    reverb_prob: 1.0
+    noise_prob: 1.0
+    noise_snr_low: 0
+    noise_snr_high: 15
+    rir_scale_factor: 1.0
+
+
+augment_pipeline: [
+    !ref <augment_wavedrop>,
+    !ref <augment_speed>,
+    !ref <add_rev>,
+    !ref <add_noise>,
+    !ref <add_rev_noise>
+]
+
+
+modules:
+    augment_wavedrop: !ref <augment_wavedrop>
+    augment_speed: !ref <augment_speed>
+    add_rev: !ref <add_rev>
+    add_noise: !ref <add_noise>
+    add_rev_noise: !ref <add_rev_noise>
+    slu_enc: !ref <slu_enc>
+    output_emb: !ref <output_emb>
+    dec: !ref <dec>
+    seq_lin: !ref <seq_lin>
+
+model: !new:torch.nn.ModuleList
+    - [!ref <slu_enc>, !ref <output_emb>,
+       !ref <dec>, !ref <seq_lin>]
+
+tokenizer: !new:sentencepiece.SentencePieceProcessor
+
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+    collect_in: !ref <save_folder>/FSC_tokenizer
+    loadables:
+        tokenizer: !ref <tokenizer>
+    paths:
+        tokenizer: !ref <tokenizer_file>
+
+beam_searcher: !new:speechbrain.decoders.S2SRNNBeamSearcher
+    embedding: !ref <output_emb>
+    decoder: !ref <dec>
+    linear: !ref <seq_lin>
+    bos_index: !ref <bos_index>
+    eos_index: !ref <eos_index>
+    min_decode_ratio: !ref <min_decode_ratio>
+    max_decode_ratio: !ref <max_decode_ratio>
+    beam_size: !ref <slu_beam_size>
+    eos_threshold: !ref <eos_threshold>
+    temperature: !ref <temperature>
+    using_max_attn_shift: False
+    max_attn_shift: 30
+    coverage_penalty: 0.
+
+opt_class: !name:torch.optim.Adam
+    lr: !ref <lr>
+
+lr_annealing: !new:speechbrain.nnet.schedulers.NewBobScheduler
+    initial_value: !ref <lr>
+    improvement_threshold: 0.0025
+    annealing_factor: 0.8
+    patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+    checkpoints_dir: !ref <save_folder>
+    recoverables:
+        model: !ref <model>
+        scheduler: !ref <lr_annealing>
+        counter: !ref <epoch_counter>
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+    apply_log: True
+
+seq_cost: !name:speechbrain.nnet.losses.nll_loss
+    label_smoothing: 0.1
+
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+    save_file: !ref <train_log>
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+    split_tokens: True

fluent-speech-commands/direct/prepare.py

@@ -0,0 +1 @@
+../prepare.py

fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/asr.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/83e944252a91fe1d0883daa1e87077df4d64c35fffb45e22fff924faace4a59c.7fdf4aabd8400c69a6228ccc17c83b7a8ebf34c5d76f23497b7cf0d7a1baaea3

fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/hyperparams.yaml

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/7aac72d39109ee19b4004d94239c2924caf33de6d85b0aff9296d844982210cb.d14310ea63844fb38520a592ea3a92e4f131b5f4683f8fa08e27b1e403c92293

fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/lm.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/651df066b5d0b2efef7208f51df93d3a0a65bedc3a3a2500cd7b8faf064e631e.b438b9af3f549a23c4458bb066c11cd51dc1cfe9bfef30d3eb66b472e93b1e8c

fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/normalizer.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/e733854cce680bcb58ce4b86bacb3cab5222880933b7b85ab17758aa5b10e9da.587fb748e80e719ed5721d5e0098c5feb2a901017135271ce2b2c6baea7e9f6e

fluent-speech-commands/direct/pretrained_models/EncoderDecoderASR-6406358104753086746/tokenizer.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/f39208eba495042a59a8404b5703ca08a39a85e4d2bf707e197b90a3323f92ab.cd7af7ea8cfcfbf0f6dd61514c361972eb82b3b76f12b0e9ee0b371f36fdc078

fluent-speech-commands/direct/results/BPE51/112011/env.log

@@ -0,0 +1,434 @@
+SpeechBrain system description
+==============================
+Python version:
+3.7.12 (default, Jan 15 2022, 18:48:18) 
+[GCC 7.5.0]
+==============================
+Installed Python packages:
+absl-py==1.0.0
+aiohttp==3.8.1
+aiosignal==1.2.0
+alabaster==0.7.12
+albumentations==0.1.12
+altair==4.2.0
+appdirs==1.4.4
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+arviz==0.11.4
+astor==0.8.1
+astropy==4.3.1
+astunparse==1.6.3
+async-timeout==4.0.2
+asynctest==0.13.0
+atari-py==0.2.9
+atomicwrites==1.4.0
+attrs==21.4.0
+audioread==2.1.9
+autograd==1.3
+Babel==2.9.1
+backcall==0.2.0
+beautifulsoup4==4.6.3
+black==19.10b0
+bleach==4.1.0
+blis==0.4.1
+bokeh==2.3.3
+Bottleneck==1.3.2
+branca==0.4.2
+bs4==0.0.1
+CacheControl==0.12.10
+cached-property==1.5.2
+cachetools==4.2.4
+catalogue==1.0.0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+cftime==1.5.2
+chardet==3.0.4
+charset-normalizer==2.0.11
+click==7.1.2
+cloudpickle==1.3.0
+cmake==3.12.0
+cmdstanpy==0.9.5
+colorcet==3.0.0
+colorlover==0.3.0
+community==1.0.0b1
+contextlib2==0.5.5
+convertdate==2.4.0
+coverage==3.7.1
+coveralls==0.5
+crcmod==1.7
+cufflinks==0.17.3
+cupy-cuda111==9.4.0
+cvxopt==1.2.7
+cvxpy==1.0.31
+cycler==0.11.0
+cymem==2.0.6
+Cython==0.29.27
+daft==0.0.4
+dask==2.12.0
+datascience==0.10.6
+datasets==1.18.3
+debugpy==1.0.0
+decorator==4.4.2
+defusedxml==0.7.1
+descartes==1.1.0
+dill==0.3.4
+distlib==0.3.4
+distributed==1.25.3
+dlib @ file:///dlib-19.18.0-cp37-cp37m-linux_x86_64.whl
+dm-tree==0.1.6
+docopt==0.6.2
+docutils==0.17.1
+dopamine-rl==1.0.5
+earthengine-api==0.1.297
+easydict==1.9
+ecos==2.0.10
+editdistance==0.5.3
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz
+entrypoints==0.3
+ephem==4.1.3
+et-xmlfile==1.1.0
+fa2==0.3.5
+fastai==1.0.61
+fastdtw==0.3.4
+fastprogress==1.0.0
+fastrlock==0.8
+fbprophet==0.7.1
+feather-format==0.4.1
+filelock==3.4.2
+firebase-admin==4.4.0
+fix-yahoo-finance==0.0.22
+flake8==3.7.9
+Flask==1.1.4
+flatbuffers==2.0
+folium==0.8.3
+frozenlist==1.3.0
+fsspec==2022.1.0
+future==0.16.0
+gast==0.4.0
+GDAL==2.2.2
+gdown==4.2.1
+gensim==3.6.0
+geographiclib==1.52
+geopy==1.17.0
+gin-config==0.5.0
+glob2==0.7
+google==2.0.3
+google-api-core==1.26.3
+google-api-python-client==1.12.10
+google-auth==1.35.0
+google-auth-httplib2==0.0.4
+google-auth-oauthlib==0.4.6
+google-cloud-bigquery==1.21.0
+google-cloud-bigquery-storage==1.1.0
+google-cloud-core==1.0.3
+google-cloud-datastore==1.8.0
+google-cloud-firestore==1.7.0
+google-cloud-language==1.2.0
+google-cloud-storage==1.18.1
+google-cloud-translate==1.5.0
+google-colab @ file:///colabtools/dist/google-colab-1.0.0.tar.gz
+google-pasta==0.2.0
+google-resumable-media==0.4.1
+googleapis-common-protos==1.54.0
+googledrivedownloader==0.4
+graphviz==0.10.1
+greenlet==1.1.2
+grpcio==1.43.0
+gspread==3.4.2
+gspread-dataframe==3.0.8
+gym==0.17.3
+h5py==3.1.0
+HeapDict==1.0.1
+hijri-converter==2.2.2
+holidays==0.10.5.2
+holoviews==1.14.7
+html5lib==1.0.1
+httpimport==0.5.18
+httplib2==0.17.4
+httplib2shim==0.0.3
+huggingface-hub==0.4.0
+humanize==0.5.1
+hyperopt==0.1.2
+HyperPyYAML==1.0.0
+ideep4py==2.0.0.post3
+identify==2.4.10
+idna==2.10
+imageio==2.4.1
+imagesize==1.3.0
+imbalanced-learn==0.8.1
+imblearn==0.0
+imgaug==0.2.9
+importlib-metadata==4.10.1
+importlib-resources==5.4.0
+imutils==0.5.4
+inflect==2.1.0
+iniconfig==1.1.1
+intel-openmp==2022.0.2
+intervaltree==2.1.0
+ipykernel==4.10.1
+ipython==5.5.0
+ipython-genutils==0.2.0
+ipython-sql==0.3.9
+ipywidgets==7.6.5
+itsdangerous==1.1.0
+jax==0.2.25
+jaxlib @ https://storage.googleapis.com/jax-releases/cuda111/jaxlib-0.1.71+cuda111-cp37-none-manylinux2010_x86_64.whl
+jedi==0.18.1
+jieba==0.42.1
+Jinja2==2.11.3
+joblib==1.1.0
+jpeg4py==0.1.4
+jsonschema==4.3.3
+jupyter==1.0.0
+jupyter-client==5.3.5
+jupyter-console==5.2.0
+jupyter-core==4.9.1
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.2
+kaggle==1.5.12
+kapre==0.3.7
+keras==2.7.0
+Keras-Preprocessing==1.1.2
+keras-vis==0.4.1
+kiwisolver==1.3.2
+korean-lunar-calendar==0.2.1
+libclang==13.0.0
+librosa==0.9.0
+lightgbm==2.2.3
+llvmlite==0.34.0
+lmdb==0.99
+LunarCalendar==0.0.9
+lxml==4.2.6
+Markdown==3.3.6
+MarkupSafe==2.0.1
+matplotlib==3.2.2
+matplotlib-inline==0.1.3
+matplotlib-venn==0.11.6
+mccabe==0.6.1
+missingno==0.5.0
+mistune==0.8.4
+mizani==0.6.0
+mkl==2019.0
+mlxtend==0.14.0
+more-itertools==8.12.0
+moviepy==0.2.3.5
+mpmath==1.2.1
+msgpack==1.0.3
+multidict==6.0.2
+multiprocess==0.70.12.2
+multitasking==0.0.10
+murmurhash==1.0.6
+music21==5.5.0
+natsort==5.5.0
+nbclient==0.5.10
+nbconvert==5.6.1
+nbformat==5.1.3
+nest-asyncio==1.5.4
+netCDF4==1.5.8
+networkx==2.6.3
+nibabel==3.0.2
+nltk==3.2.5
+nodeenv==1.6.0
+notebook==5.3.1
+numba==0.51.2
+numexpr==2.8.1
+numpy==1.19.5
+nvidia-ml-py3==7.352.0
+oauth2client==4.1.3
+oauthlib==3.2.0
+okgrade==0.4.3
+opencv-contrib-python==4.1.2.30
+opencv-python==4.1.2.30
+openpyxl==3.0.9
+opt-einsum==3.3.0
+osqp==0.6.2.post0
+packaging==21.3
+palettable==3.3.0
+pandas==1.3.5
+pandas-datareader==0.9.0
+pandas-gbq==0.13.3
+pandas-profiling==1.4.1
+pandocfilters==1.5.0
+panel==0.12.1
+param==1.12.0
+parso==0.8.3
+pathlib==1.0.1
+pathspec==0.9.0
+patsy==0.5.2
+pep517==0.12.0
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==7.1.2
+pip-tools==6.2.0
+plac==1.1.3
+platformdirs==2.5.0
+plotly==5.5.0
+plotnine==0.6.0
+pluggy==0.13.1
+pooch==1.6.0
+portpicker==1.3.9
+pre-commit==2.17.0
+prefetch-generator==1.0.1
+preshed==3.0.6
+prettytable==3.0.0
+progressbar2==3.38.0
+prometheus-client==0.13.1
+promise==2.3
+prompt-toolkit==1.0.18
+protobuf==3.17.3
+psutil==5.4.8
+psycopg2==2.7.6.1
+ptyprocess==0.7.0
+py==1.11.0
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycocotools==2.0.4
+pycodestyle==2.5.0
+pycparser==2.21
+pyct==0.4.8
+pydata-google-auth==1.3.0
+pydot==1.3.0
+pydot-ng==2.0.0
+pydotplus==2.0.2
+PyDrive==1.3.1
+pyemd==0.5.1
+pyerfa==2.0.0.1
+pyflakes==2.1.1
+pyglet==1.5.0
+Pygments==2.6.1
+pygobject==3.26.1
+pymc3==3.11.4
+PyMeeus==0.5.11
+pymongo==4.0.1
+pymystem3==0.2.0
+PyOpenGL==3.1.5
+pyparsing==3.0.7
+pyrsistent==0.18.1
+pysndfile==1.3.8
+PySocks==1.7.1
+pystan==2.19.1.1
+pytest==5.4.1
+python-apt==0.0.0
+python-chess==0.23.11
+python-dateutil==2.8.2
+python-louvain==0.16
+python-slugify==5.0.2
+python-utils==3.1.0
+pytz==2018.9
+pyviz-comms==2.1.0
+PyWavelets==1.2.0
+PyYAML==6.0
+pyzmq==22.3.0
+qdldl==0.1.5.post0
+qtconsole==5.2.2
+QtPy==2.0.1
+regex==2019.12.20
+requests==2.23.0
+requests-oauthlib==1.3.1
+resampy==0.2.2
+rpy2==3.4.5
+rsa==4.8
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-image==0.18.3
+scikit-learn==1.0.2
+scipy==1.4.1
+screen-resolution-extra==0.0.0
+scs==3.1.0
+seaborn==0.11.2
+semver==2.13.0
+Send2Trash==1.8.0
+sentencepiece==0.1.96
+setuptools-git==1.2
+Shapely==1.8.0
+simplegeneric==0.8.1
+six==1.15.0
+sklearn==0.0
+sklearn-pandas==1.8.0
+smart-open==5.2.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+SoundFile==0.10.3.post1
+spacy==2.2.4
+speechbrain==0.5.11
+Sphinx==1.8.6
+sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-websupport==1.2.4
+SQLAlchemy==1.4.31
+sqlparse==0.4.2
+srsly==1.0.5
+statsmodels==0.10.2
+sympy==1.7.1
+tables==3.7.0
+tabulate==0.8.9
+tblib==1.7.0
+tenacity==8.0.1
+tensorboard==2.7.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow @ file:///tensorflow-2.7.0-cp37-cp37m-linux_x86_64.whl
+tensorflow-datasets==4.0.1
+tensorflow-estimator==2.7.0
+tensorflow-gcs-config==2.7.0
+tensorflow-hub==0.12.0
+tensorflow-io-gcs-filesystem==0.24.0
+tensorflow-metadata==1.6.0
+tensorflow-probability==0.15.0
+termcolor==1.1.0
+terminado==0.13.1
+testpath==0.5.0
+text-unidecode==1.3
+textblob==0.15.3
+Theano-PyMC==1.1.2
+thinc==7.4.0
+threadpoolctl==3.1.0
+tifffile==2021.11.2
+tokenizers==0.11.4
+toml==0.10.2
+tomli==2.0.0
+toolz==0.11.2
+torch @ https://download.pytorch.org/whl/cu111/torch-1.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl
+torchaudio @ https://download.pytorch.org/whl/cu111/torchaudio-0.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl
+torchsummary==1.5.1
+torchtext==0.11.0
+torchvision @ https://download.pytorch.org/whl/cu111/torchvision-0.11.1%2Bcu111-cp37-cp37m-linux_x86_64.whl
+tornado==5.1.1
+tqdm==4.62.3
+traitlets==5.1.1
+transformers==4.16.2
+tweepy==3.10.0
+typed-ast==1.5.2
+typeguard==2.7.1
+typing-extensions==3.10.0.2
+tzlocal==1.5.1
+uritemplate==3.0.1
+urllib3==1.24.3
+vega-datasets==0.9.0
+virtualenv==20.13.1
+wasabi==0.9.0
+wcwidth==0.2.5
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.2
+wordcloud==1.5.0
+wrapt==1.13.3
+xarray==0.18.2
+xgboost==0.90
+xkit==0.0.0
+xlrd==1.1.0
+xlwt==1.3.0
+xxhash==2.0.2
+yamllint==1.23.0
+yarl==1.7.2
+yellowbrick==1.3.post1
+zict==2.0.0
+zipp==3.7.0
+==============================
+Git revision:
+9d56d508
+==============================
+Cuda version:
+11.1

fluent-speech-commands/direct/results/BPE51/112011/hyperparams.yaml

@@ -0,0 +1,200 @@
+# Generated 2022-02-15 from:
+# /content/speechbrain/recipes/fluent-speech-commands/direct/hparams/train.yaml
+# yamllint disable
+# ############################################################################
+# Model: Direct SLU
+# Encoder: Pre-trained ASR encoder -> LSTM
+# Decoder: GRU + beamsearch
+# Tokens: BPE with unigram
+# losses: NLL
+# Training: Fluent Speech Commands
+# Authors:  Loren Lugosch, Mirco Ravanelli 2020
+# ############################################################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 112011
+__set_seed: !apply:torch.manual_seed [112011]
+output_folder: results/BPE51/112011
+save_folder: results/BPE51/112011/save
+train_log: results/BPE51/112011/train_log.txt
+
+# Data files
+data_folder: /content/fluent_speech_commands_dataset
+                          # e.g, /localscratch/fluent_speech_commands_dataset
+rir_folder: /content/fluent_speech_commands_dataset # Change it if needed
+csv_train: results/BPE51/112011/train.csv
+csv_valid: results/BPE51/112011/valid.csv
+csv_test: results/BPE51/112011/test.csv
+tokenizer_file: https://www.dropbox.com/s/hvf2huofnq0sjbn/51_unigram.model?dl=1
+skip_prep: false
+# Training parameters
+number_of_epochs: 4
+batch_size: 8
+lr: 0.0003
+token_type: unigram # ["unigram", "bpe", "char"]
+sorting: random
+
+# Model parameters
+sample_rate: 16000
+emb_size: 128
+dec_neurons: 512
+output_neurons: 51 # index(eos/bos) = 0
+ASR_encoder_dim: 512
+encoder_dim: 256
+
+# Decoding parameters
+bos_index: 0
+eos_index: 0
+min_decode_ratio: 0.0
+max_decode_ratio: 10.0
+slu_beam_size: 80
+eos_threshold: 1.5
+temperature: 1.25
+
+dataloader_opts:
+  batch_size: 8
+  shuffle: true
+
+epoch_counter: &id013 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 4
+
+# Models
+asr_model: !apply:speechbrain.pretrained.EncoderDecoderASR.from_hparams
+  source: speechbrain/asr-crdnn-rnnlm-librispeech
+  run_opts: {device: cuda:0}
+
+slu_enc: &id006 !new:speechbrain.nnet.containers.Sequential
+  input_shape: [null, null, 512]
+  lstm: !new:speechbrain.nnet.RNN.LSTM
+    input_size: 512
+    bidirectional: true
+    hidden_size: 256
+    num_layers: 2
+  linear: !new:speechbrain.nnet.linear.Linear
+    input_size: 512
+    n_neurons: 256
+
+output_emb: &id007 !new:speechbrain.nnet.embedding.Embedding
+  num_embeddings: 51
+  embedding_dim: 128
+
+dec: &id008 !new:speechbrain.nnet.RNN.AttentionalRNNDecoder
+  enc_dim: 256
+  input_size: 128
+  rnn_type: gru
+  attn_type: keyvalue
+  hidden_size: 512
+  attn_dim: 512
+  num_layers: 3
+  scaling: 1.0
+  dropout: 0.0
+
+seq_lin: &id009 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 512
+  n_neurons: 51
+
+augment_wavedrop: &id001 !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [100]
+
+augment_speed: &id002 !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [95, 100, 105]
+
+add_rev: &id003 !new:speechbrain.lobes.augment.EnvCorrupt
+  openrir_folder: /content/fluent_speech_commands_dataset
+  openrir_max_noise_len: 3.0    # seconds
+  reverb_prob: 1.0
+  noise_prob: 0.0
+  noise_snr_low: 0
+  noise_snr_high: 15
+  rir_scale_factor: 1.0
+
+add_noise: &id004 !new:speechbrain.lobes.augment.EnvCorrupt
+  openrir_folder: /content/fluent_speech_commands_dataset
+  openrir_max_noise_len: 3.0    # seconds
+  reverb_prob: 0.0
+  noise_prob: 1.0
+  noise_snr_low: 0
+  noise_snr_high: 15
+  rir_scale_factor: 1.0
+
+add_rev_noise: &id005 !new:speechbrain.lobes.augment.EnvCorrupt
+  openrir_folder: /content/fluent_speech_commands_dataset
+  openrir_max_noise_len: 3.0    # seconds
+  reverb_prob: 1.0
+  noise_prob: 1.0
+  noise_snr_low: 0
+  noise_snr_high: 15
+  rir_scale_factor: 1.0
+
+
+augment_pipeline: [*id001, *id002, *id003, *id004, *id005]
+
+
+modules:
+  augment_wavedrop: *id001
+  augment_speed: *id002
+  add_rev: *id003
+  add_noise: *id004
+  add_rev_noise: *id005
+  slu_enc: *id006
+  output_emb: *id007
+  dec: *id008
+  seq_lin: *id009
+model: &id011 !new:torch.nn.ModuleList
+- [*id006, *id007, *id008, *id009]
+tokenizer: &id010 !new:sentencepiece.SentencePieceProcessor
+
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+  collect_in: results/BPE51/112011/save/FSC_tokenizer
+  loadables:
+    tokenizer: *id010
+  paths:
+    tokenizer: https://www.dropbox.com/s/hvf2huofnq0sjbn/51_unigram.model?dl=1
+
+beam_searcher: !new:speechbrain.decoders.S2SRNNBeamSearcher
+  embedding: *id007
+  decoder: *id008
+  linear: *id009
+  bos_index: 0
+  eos_index: 0
+  min_decode_ratio: 0.0
+  max_decode_ratio: 10.0
+  beam_size: 80
+  eos_threshold: 1.5
+  temperature: 1.25
+  using_max_attn_shift: false
+  max_attn_shift: 30
+  coverage_penalty: 0.
+
+opt_class: !name:torch.optim.Adam
+  lr: 0.0003
+
+lr_annealing: &id012 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0003
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: results/BPE51/112011/save
+  recoverables:
+    model: *id011
+    scheduler: *id012
+    counter: *id013
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+seq_cost: !name:speechbrain.nnet.losses.nll_loss
+  label_smoothing: 0.1
+
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: results/BPE51/112011/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

fluent-speech-commands/direct/results/BPE51/112011/log.txt

@@ -0,0 +1,454 @@
+2022-02-15 02:15:13,142 - speechbrain.core - INFO - Beginning experiment!
+2022-02-15 02:15:13,142 - speechbrain.core - INFO - Experiment folder: results/BPE51/112011
+2022-02-15 02:15:14,668 - speechbrain.utils.superpowers - DEBUG - absl-py==1.0.0
+aiohttp==3.8.1
+aiosignal==1.2.0
+alabaster==0.7.12
+albumentations==0.1.12
+altair==4.2.0
+appdirs==1.4.4
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+arviz==0.11.4
+astor==0.8.1
+astropy==4.3.1
+astunparse==1.6.3
+async-timeout==4.0.2
+asynctest==0.13.0
+atari-py==0.2.9
+atomicwrites==1.4.0
+attrs==21.4.0
+audioread==2.1.9
+autograd==1.3
+Babel==2.9.1
+backcall==0.2.0
+beautifulsoup4==4.6.3
+black==19.10b0
+bleach==4.1.0
+blis==0.4.1
+bokeh==2.3.3
+Bottleneck==1.3.2
+branca==0.4.2
+bs4==0.0.1
+CacheControl==0.12.10
+cached-property==1.5.2
+cachetools==4.2.4
+catalogue==1.0.0
+certifi==2021.10.8
+cffi==1.15.0
+cfgv==3.3.1
+cftime==1.5.2
+chardet==3.0.4
+charset-normalizer==2.0.11
+click==7.1.2
+cloudpickle==1.3.0
+cmake==3.12.0
+cmdstanpy==0.9.5
+colorcet==3.0.0
+colorlover==0.3.0
+community==1.0.0b1
+contextlib2==0.5.5
+convertdate==2.4.0
+coverage==3.7.1
+coveralls==0.5
+crcmod==1.7
+cufflinks==0.17.3
+cupy-cuda111==9.4.0
+cvxopt==1.2.7
+cvxpy==1.0.31
+cycler==0.11.0
+cymem==2.0.6
+Cython==0.29.27
+daft==0.0.4
+dask==2.12.0
+datascience==0.10.6
+datasets==1.18.3
+debugpy==1.0.0
+decorator==4.4.2
+defusedxml==0.7.1
+descartes==1.1.0
+dill==0.3.4
+distlib==0.3.4
+distributed==1.25.3
+dlib @ file:///dlib-19.18.0-cp37-cp37m-linux_x86_64.whl
+dm-tree==0.1.6
+docopt==0.6.2
+docutils==0.17.1
+dopamine-rl==1.0.5
+earthengine-api==0.1.297
+easydict==1.9
+ecos==2.0.10
+editdistance==0.5.3
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz
+entrypoints==0.3
+ephem==4.1.3
+et-xmlfile==1.1.0
+fa2==0.3.5
+fastai==1.0.61
+fastdtw==0.3.4
+fastprogress==1.0.0
+fastrlock==0.8
+fbprophet==0.7.1
+feather-format==0.4.1
+filelock==3.4.2
+firebase-admin==4.4.0
+fix-yahoo-finance==0.0.22
+flake8==3.7.9
+Flask==1.1.4
+flatbuffers==2.0
+folium==0.8.3
+frozenlist==1.3.0
+fsspec==2022.1.0
+future==0.16.0
+gast==0.4.0
+GDAL==2.2.2
+gdown==4.2.1
+gensim==3.6.0
+geographiclib==1.52
+geopy==1.17.0
+gin-config==0.5.0
+glob2==0.7
+google==2.0.3
+google-api-core==1.26.3
+google-api-python-client==1.12.10
+google-auth==1.35.0
+google-auth-httplib2==0.0.4
+google-auth-oauthlib==0.4.6
+google-cloud-bigquery==1.21.0
+google-cloud-bigquery-storage==1.1.0
+google-cloud-core==1.0.3
+google-cloud-datastore==1.8.0
+google-cloud-firestore==1.7.0
+google-cloud-language==1.2.0
+google-cloud-storage==1.18.1
+google-cloud-translate==1.5.0
+google-colab @ file:///colabtools/dist/google-colab-1.0.0.tar.gz
+google-pasta==0.2.0
+google-resumable-media==0.4.1
+googleapis-common-protos==1.54.0
+googledrivedownloader==0.4
+graphviz==0.10.1
+greenlet==1.1.2
+grpcio==1.43.0
+gspread==3.4.2
+gspread-dataframe==3.0.8
+gym==0.17.3
+h5py==3.1.0
+HeapDict==1.0.1
+hijri-converter==2.2.2
+holidays==0.10.5.2
+holoviews==1.14.7
+html5lib==1.0.1
+httpimport==0.5.18
+httplib2==0.17.4
+httplib2shim==0.0.3
+huggingface-hub==0.4.0
+humanize==0.5.1
+hyperopt==0.1.2
+HyperPyYAML==1.0.0
+ideep4py==2.0.0.post3
+identify==2.4.10
+idna==2.10
+imageio==2.4.1
+imagesize==1.3.0
+imbalanced-learn==0.8.1
+imblearn==0.0
+imgaug==0.2.9
+importlib-metadata==4.10.1
+importlib-resources==5.4.0
+imutils==0.5.4
+inflect==2.1.0
+iniconfig==1.1.1
+intel-openmp==2022.0.2
+intervaltree==2.1.0
+ipykernel==4.10.1
+ipython==5.5.0
+ipython-genutils==0.2.0
+ipython-sql==0.3.9
+ipywidgets==7.6.5
+itsdangerous==1.1.0
+jax==0.2.25
+jaxlib @ https://storage.googleapis.com/jax-releases/cuda111/jaxlib-0.1.71+cuda111-cp37-none-manylinux2010_x86_64.whl
+jedi==0.18.1
+jieba==0.42.1
+Jinja2==2.11.3
+joblib==1.1.0
+jpeg4py==0.1.4
+jsonschema==4.3.3
+jupyter==1.0.0
+jupyter-client==5.3.5
+jupyter-console==5.2.0
+jupyter-core==4.9.1
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.2
+kaggle==1.5.12
+kapre==0.3.7
+keras==2.7.0
+Keras-Preprocessing==1.1.2
+keras-vis==0.4.1
+kiwisolver==1.3.2
+korean-lunar-calendar==0.2.1
+libclang==13.0.0
+librosa==0.9.0
+lightgbm==2.2.3
+llvmlite==0.34.0
+lmdb==0.99
+LunarCalendar==0.0.9
+lxml==4.2.6
+Markdown==3.3.6
+MarkupSafe==2.0.1
+matplotlib==3.2.2
+matplotlib-inline==0.1.3
+matplotlib-venn==0.11.6
+mccabe==0.6.1
+missingno==0.5.0
+mistune==0.8.4
+mizani==0.6.0
+mkl==2019.0
+mlxtend==0.14.0
+more-itertools==8.12.0
+moviepy==0.2.3.5
+mpmath==1.2.1
+msgpack==1.0.3
+multidict==6.0.2
+multiprocess==0.70.12.2
+multitasking==0.0.10
+murmurhash==1.0.6
+music21==5.5.0
+natsort==5.5.0
+nbclient==0.5.10
+nbconvert==5.6.1
+nbformat==5.1.3
+nest-asyncio==1.5.4
+netCDF4==1.5.8
+networkx==2.6.3
+nibabel==3.0.2
+nltk==3.2.5
+nodeenv==1.6.0
+notebook==5.3.1
+numba==0.51.2
+numexpr==2.8.1
+numpy==1.19.5
+nvidia-ml-py3==7.352.0
+oauth2client==4.1.3
+oauthlib==3.2.0
+okgrade==0.4.3
+opencv-contrib-python==4.1.2.30
+opencv-python==4.1.2.30
+openpyxl==3.0.9
+opt-einsum==3.3.0
+osqp==0.6.2.post0
+packaging==21.3
+palettable==3.3.0
+pandas==1.3.5
+pandas-datareader==0.9.0
+pandas-gbq==0.13.3
+pandas-profiling==1.4.1
+pandocfilters==1.5.0
+panel==0.12.1
+param==1.12.0
+parso==0.8.3
+pathlib==1.0.1
+pathspec==0.9.0
+patsy==0.5.2
+pep517==0.12.0
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==7.1.2
+pip-tools==6.2.0
+plac==1.1.3
+platformdirs==2.5.0
+plotly==5.5.0
+plotnine==0.6.0
+pluggy==0.13.1
+pooch==1.6.0
+portpicker==1.3.9
+pre-commit==2.17.0
+prefetch-generator==1.0.1
+preshed==3.0.6
+prettytable==3.0.0
+progressbar2==3.38.0
+prometheus-client==0.13.1
+promise==2.3
+prompt-toolkit==1.0.18
+protobuf==3.17.3
+psutil==5.4.8
+psycopg2==2.7.6.1
+ptyprocess==0.7.0
+py==1.11.0
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycocotools==2.0.4
+pycodestyle==2.5.0
+pycparser==2.21
+pyct==0.4.8
+pydata-google-auth==1.3.0
+pydot==1.3.0
+pydot-ng==2.0.0
+pydotplus==2.0.2
+PyDrive==1.3.1
+pyemd==0.5.1
+pyerfa==2.0.0.1
+pyflakes==2.1.1
+pyglet==1.5.0
+Pygments==2.6.1
+pygobject==3.26.1
+pymc3==3.11.4
+PyMeeus==0.5.11
+pymongo==4.0.1
+pymystem3==0.2.0
+PyOpenGL==3.1.5
+pyparsing==3.0.7
+pyrsistent==0.18.1
+pysndfile==1.3.8
+PySocks==1.7.1
+pystan==2.19.1.1
+pytest==5.4.1
+python-apt==0.0.0
+python-chess==0.23.11
+python-dateutil==2.8.2
+python-louvain==0.16
+python-slugify==5.0.2
+python-utils==3.1.0
+pytz==2018.9
+pyviz-comms==2.1.0
+PyWavelets==1.2.0
+PyYAML==6.0
+pyzmq==22.3.0
+qdldl==0.1.5.post0
+qtconsole==5.2.2
+QtPy==2.0.1
+regex==2019.12.20
+requests==2.23.0
+requests-oauthlib==1.3.1
+resampy==0.2.2
+rpy2==3.4.5
+rsa==4.8
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+sacremoses==0.0.47
+scikit-image==0.18.3
+scikit-learn==1.0.2
+scipy==1.4.1
+screen-resolution-extra==0.0.0
+scs==3.1.0
+seaborn==0.11.2
+semver==2.13.0
+Send2Trash==1.8.0
+sentencepiece==0.1.96
+setuptools-git==1.2
+Shapely==1.8.0
+simplegeneric==0.8.1
+six==1.15.0
+sklearn==0.0
+sklearn-pandas==1.8.0
+smart-open==5.2.1
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+SoundFile==0.10.3.post1
+spacy==2.2.4
+speechbrain==0.5.11
+Sphinx==1.8.6
+sphinxcontrib-serializinghtml==1.1.5
+sphinxcontrib-websupport==1.2.4
+SQLAlchemy==1.4.31
+sqlparse==0.4.2
+srsly==1.0.5
+statsmodels==0.10.2
+sympy==1.7.1
+tables==3.7.0
+tabulate==0.8.9
+tblib==1.7.0
+tenacity==8.0.1
+tensorboard==2.7.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow @ file:///tensorflow-2.7.0-cp37-cp37m-linux_x86_64.whl
+tensorflow-datasets==4.0.1
+tensorflow-estimator==2.7.0
+tensorflow-gcs-config==2.7.0
+tensorflow-hub==0.12.0
+tensorflow-io-gcs-filesystem==0.24.0
+tensorflow-metadata==1.6.0
+tensorflow-probability==0.15.0
+termcolor==1.1.0
+terminado==0.13.1
+testpath==0.5.0
+text-unidecode==1.3
+textblob==0.15.3
+Theano-PyMC==1.1.2
+thinc==7.4.0
+threadpoolctl==3.1.0
+tifffile==2021.11.2
+tokenizers==0.11.4
+toml==0.10.2
+tomli==2.0.0
+toolz==0.11.2
+torch @ https://download.pytorch.org/whl/cu111/torch-1.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl
+torchaudio @ https://download.pytorch.org/whl/cu111/torchaudio-0.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl
+torchsummary==1.5.1
+torchtext==0.11.0
+torchvision @ https://download.pytorch.org/whl/cu111/torchvision-0.11.1%2Bcu111-cp37-cp37m-linux_x86_64.whl
+tornado==5.1.1
+tqdm==4.62.3
+traitlets==5.1.1
+transformers==4.16.2
+tweepy==3.10.0
+typed-ast==1.5.2
+typeguard==2.7.1
+typing-extensions==3.10.0.2
+tzlocal==1.5.1
+uritemplate==3.0.1
+urllib3==1.24.3
+vega-datasets==0.9.0
+virtualenv==20.13.1
+wasabi==0.9.0
+wcwidth==0.2.5
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.2
+wordcloud==1.5.0
+wrapt==1.13.3
+xarray==0.18.2
+xgboost==0.90
+xkit==0.0.0
+xlrd==1.1.0
+xlwt==1.3.0
+xxhash==2.0.2
+yamllint==1.23.0
+yarl==1.7.2
+yellowbrick==1.3.post1
+zict==2.0.0
+zipp==3.7.0
+
+
+2022-02-15 02:15:14,791 - speechbrain.utils.superpowers - DEBUG - 9d56d508
+
+
+2022-02-15 02:15:14,794 - prepare - INFO - Preparing results/BPE51/112011/train.csv...
+2022-02-15 02:15:51,789 - prepare - INFO - Preparing results/BPE51/112011/valid.csv...
+2022-02-15 02:15:54,371 - prepare - INFO - Preparing results/BPE51/112011/test.csv...
+2022-02-15 02:15:57,696 - speechbrain.utils.parameter_transfer - DEBUG - Collecting files (or symlinks) for pretraining in results/BPE51/112011/save/FSC_tokenizer.
+2022-02-15 02:15:57,696 - speechbrain.pretrained.fetching - INFO - Fetch 51_unigram.model?dl=1: Downloading from normal URL https://www.dropbox.com/s/hvf2huofnq0sjbn/51_unigram.model?dl=1.
+2022-02-15 02:15:58,662 - speechbrain.utils.parameter_transfer - INFO - Loading pretrained files for: tokenizer
+2022-02-15 02:15:58,689 - speechbrain.core - INFO - 9.3M trainable parameters in SLU
+2022-02-15 02:15:58,690 - speechbrain.utils.checkpoints - INFO - Would load a checkpoint here, but none found yet.
+2022-02-15 02:15:58,690 - speechbrain.utils.epoch_loop - INFO - Going into epoch 1
+2022-02-15 02:55:37,944 - speechbrain.utils.train_logger - INFO - epoch: 1, lr: 3.00e-04 - train loss: 7.70e-01 - valid loss: 7.10e-01, valid CER: 3.82e-01, valid WER: 1.01
+2022-02-15 02:55:38,203 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+02-55-37+00
+2022-02-15 02:55:38,208 - speechbrain.utils.epoch_loop - INFO - Going into epoch 2
+2022-02-15 03:34:48,102 - speechbrain.utils.train_logger - INFO - epoch: 2, lr: 3.00e-04 - train loss: 7.10e-01 - valid loss: 7.08e-01, valid CER: 3.45e-01, valid WER: 8.11e-01
+2022-02-15 03:34:48,368 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+03-34-48+00
+2022-02-15 03:34:48,400 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+02-55-37+00
+2022-02-15 03:34:48,401 - speechbrain.utils.epoch_loop - INFO - Going into epoch 3
+2022-02-15 04:14:03,140 - speechbrain.utils.train_logger - INFO - epoch: 3, lr: 3.00e-04 - train loss: 7.08e-01 - valid loss: 7.07e-01, valid CER: 2.30e-01, valid WER: 5.79e-01
+2022-02-15 04:14:03,373 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00
+2022-02-15 04:14:03,429 - speechbrain.utils.checkpoints - INFO - Deleted checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+03-34-48+00
+2022-02-15 04:14:03,430 - speechbrain.utils.epoch_loop - INFO - Going into epoch 4
+2022-02-15 04:53:03,184 - speechbrain.nnet.schedulers - INFO - Changing lr from 0.0003 to 0.00024
+2022-02-15 04:53:03,184 - speechbrain.utils.train_logger - INFO - epoch: 4, lr: 3.00e-04 - train loss: 7.08e-01 - valid loss: 7.07e-01, valid CER: 3.17e-01, valid WER: 7.26e-01
+2022-02-15 04:53:03,440 - speechbrain.utils.checkpoints - INFO - Saved an end-of-epoch checkpoint in results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00
+2022-02-15 04:53:03,473 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00
+2022-02-15 04:53:03,520 - root - DEBUG - SaveableDataLoader was requested to load a checkpoint, but the DataLoader has already been iterated. The DataLoader file will be ignored. This is normal in evaluation, when a checkpoint is loaded just to retrieve the best model.
+2022-02-15 04:56:32,698 - speechbrain.utils.train_logger - INFO - Epoch loaded: 4 - test loss: 7.05e-01, test CER: 2.37e-02, test WER: 6.91e-02

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+WER: 0.5789473684210527
+end-of-epoch: true
+unixtime: 1644898443.140418

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/brain.ckpt

@@ -0,0 +1,2 @@
+avg_train_loss: 0.0
+step: 0

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/counter.ckpt

@@ -0,0 +1 @@
+3

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/dataloader-TRAIN.ckpt

@@ -0,0 +1 @@
+2892

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f0875b693545510c3de685a7b12825be269f311f47dff8e4dc09167aae8943
+size 37181975

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-14-03+00/optimizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6a1d3d05b4a82113bfece0e7dfb5a3fe11e38cd02fcc01c9e2a367cd1b0f444
+size 74367087

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/CKPT.yaml

@@ -0,0 +1,4 @@
+# yamllint disable
+WER: 0.7263157894736842
+end-of-epoch: true
+unixtime: 1644900783.1849935

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/brain.ckpt

@@ -0,0 +1,2 @@
+avg_train_loss: 0.0
+step: 0

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/counter.ckpt

@@ -0,0 +1 @@
+4

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/dataloader-TRAIN.ckpt

@@ -0,0 +1 @@
+2892

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b0d3ed61e45b003d62e451327473a9cfc56f64c15368ea8caafd09f3f1b948
+size 37181975

fluent-speech-commands/direct/results/BPE51/112011/save/CKPT+2022-02-15+04-53-03+00/optimizer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce37a00b1a444e7e8b1076ec10f430e63b8c12c77406cb3aa88353b95ab1a17f
+size 74367087

fluent-speech-commands/direct/results/BPE51/112011/train.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env/python3
+"""
+Recipe for "direct" (speech -> semantics) SLU with ASR-based transfer learning.
+
+We encode input waveforms into features using a model trained on LibriSpeech,
+then feed the features into a seq2seq model to map them to semantics.
+
+(Adapted from the LibriSpeech seq2seq ASR recipe written by Ju-Chieh Chou, Mirco Ravanelli, Abdel Heba, and Peter Plantinga.)
+
+Run using:
+> python train.py hparams/train.yaml
+
+Authors
+ * Loren Lugosch 2020
+ * Mirco Ravanelli 2020
+"""
+
+import sys
+import torch
+import speechbrain as sb
+import logging
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.utils.distributed import run_on_main
+
+logger = logging.getLogger(__name__)
+
+# Define training procedure
+
+
+class SLU(sb.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, tokens_bos_lens = batch.tokens_bos
+
+        # Add augmentation if specified
+        if stage == sb.Stage.TRAIN:
+            # Applying the augmentation pipeline
+            wavs_aug_tot = []
+            wavs_aug_tot.append(wavs)
+            for count, augment in enumerate(self.hparams.augment_pipeline):
+
+                # Apply augment
+                wavs_aug = augment(wavs, wav_lens)
+
+                # Managing speed change
+                if wavs_aug.shape[1] > wavs.shape[1]:
+                    wavs_aug = wavs_aug[:, 0 : wavs.shape[1]]
+                else:
+                    zero_sig = torch.zeros_like(wavs)
+                    zero_sig[:, 0 : wavs_aug.shape[1]] = wavs_aug
+                    wavs_aug = zero_sig
+
+                wavs_aug_tot.append(wavs_aug)
+
+            wavs = torch.cat(wavs_aug_tot, dim=0)
+            self.n_augment = len(wavs_aug_tot)
+            wav_lens = torch.cat([wav_lens] * self.n_augment)
+            tokens_bos = torch.cat([tokens_bos] * self.n_augment)
+
+        # ASR encoder forward pass
+        with torch.no_grad():
+            ASR_encoder_out = self.hparams.asr_model.encode_batch(
+                wavs.detach(), wav_lens
+            )
+
+        # SLU forward pass
+        encoder_out = self.hparams.slu_enc(ASR_encoder_out)
+        e_in = self.hparams.output_emb(tokens_bos)
+        h, _ = self.hparams.dec(e_in, encoder_out, wav_lens)
+
+        # Output layer for seq2seq log-probabilities
+        logits = self.hparams.seq_lin(h)
+        p_seq = self.hparams.log_softmax(logits)
+
+        # Compute outputs
+        if (
+            stage == sb.Stage.TRAIN
+            and self.batch_count % show_results_every != 0
+        ):
+            return p_seq, wav_lens
+        else:
+            p_tokens, scores = self.hparams.beam_searcher(encoder_out, wav_lens)
+            return p_seq, wav_lens, p_tokens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (NLL) given predictions and targets."""
+
+        if (
+            stage == sb.Stage.TRAIN
+            and self.batch_count % show_results_every != 0
+        ):
+            p_seq, wav_lens = predictions
+        else:
+            p_seq, wav_lens, predicted_tokens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        if hasattr(self.hparams, "env_corrupt") and stage == sb.Stage.TRAIN:
+            tokens_eos = torch.cat([tokens_eos, tokens_eos], dim=0)
+            tokens_eos_lens = torch.cat(
+                [tokens_eos_lens, tokens_eos_lens], dim=0
+            )
+
+        if stage == sb.Stage.TRAIN:
+            tokens_eos = torch.cat([tokens_eos] * self.n_augment, dim=0)
+            tokens_eos_lens = torch.cat(
+                [tokens_eos_lens] * self.n_augment, dim=0
+            )
+
+        loss_seq = self.hparams.seq_cost(
+            p_seq, tokens_eos, length=tokens_eos_lens
+        )
+
+        # (No ctc loss)
+        loss = loss_seq
+
+        if (stage != sb.Stage.TRAIN) or (
+            self.batch_count % show_results_every == 0
+        ):
+            # Decode token terms to words
+            predicted_semantics = [
+                tokenizer.decode_ids(utt_seq).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+
+            target_semantics = [wrd.split(" ") for wrd in batch.semantics]
+
+            for i in range(len(target_semantics)):
+                print(" ".join(predicted_semantics[i]).replace("|", ","))
+                print(" ".join(target_semantics[i]).replace("|", ","))
+                print("")
+
+            if stage != sb.Stage.TRAIN:
+                self.wer_metric.append(
+                    ids, predicted_semantics, target_semantics
+                )
+                self.cer_metric.append(
+                    ids, predicted_semantics, target_semantics
+                )
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        predictions = self.compute_forward(batch, sb.Stage.TRAIN)
+        loss = self.compute_objectives(predictions, batch, sb.Stage.TRAIN)
+        loss.backward()
+        if self.check_gradients(loss):
+            self.optimizer.step()
+        self.optimizer.zero_grad()
+        self.batch_count += 1
+        return loss.detach()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        self.batch_count = 0
+
+        if stage != sb.Stage.TRAIN:
+
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of a epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr, new_lr = self.hparams.lr_annealing(stage_stats["WER"])
+            sb.nnet.schedulers.update_learning_rate(self.optimizer, new_lr)
+            self.hparams.train_logger.log_stats(
+                stats_meta={"epoch": epoch, "lr": old_lr},
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_train"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(sort_key="duration")
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration", reverse=True
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_valid"], replacements={"data_root": data_folder},
+    )
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_test"], replacements={"data_root": data_folder},
+    )
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    tokenizer = hparams["tokenizer"]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        sig = sb.dataio.dataio.read_audio(wav)
+        return sig
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("semantics")
+    @sb.utils.data_pipeline.provides(
+        "semantics", "token_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(semantics):
+        yield semantics
+        tokens_list = tokenizer.encode_as_ids(semantics)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets,
+        ["id", "sig", "semantics", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data, tokenizer
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    show_results_every = 100  # plots results every N iterations
+
+    # If distributed_launch=True then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Dataset prep
+    from prepare import prepare_FSC  # noqa
+
+    # multi-gpu (ddp) save data preparation
+    run_on_main(
+        prepare_FSC,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["output_folder"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # here we create the datasets objects as well as tokenization and encoding
+    (train_set, valid_set, test_set, tokenizer,) = dataio_prepare(hparams)
+
+    # We download and pretrain the tokenizer
+    run_on_main(hparams["pretrainer"].collect_files)
+    hparams["pretrainer"].load_collected(device=run_opts["device"])
+
+    # Brain class initialization
+    slu_brain = SLU(
+        modules=hparams["modules"],
+        opt_class=hparams["opt_class"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # adding objects to trainer:
+    slu_brain.tokenizer = tokenizer
+
+    # Training
+    slu_brain.fit(
+        slu_brain.hparams.epoch_counter,
+        train_set,
+        valid_set,
+        train_loader_kwargs=hparams["dataloader_opts"],
+        valid_loader_kwargs=hparams["dataloader_opts"],
+    )
+
+    # Test
+    slu_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    slu_brain.evaluate(test_set, test_loader_kwargs=hparams["dataloader_opts"])

fluent-speech-commands/direct/results/BPE51/112011/train_log.txt

@@ -0,0 +1,5 @@
+epoch: 1, lr: 3.00e-04 - train loss: 7.70e-01 - valid loss: 7.10e-01, valid CER: 3.82e-01, valid WER: 1.01
+epoch: 2, lr: 3.00e-04 - train loss: 7.10e-01 - valid loss: 7.08e-01, valid CER: 3.45e-01, valid WER: 8.11e-01
+epoch: 3, lr: 3.00e-04 - train loss: 7.08e-01 - valid loss: 7.07e-01, valid CER: 2.30e-01, valid WER: 5.79e-01
+epoch: 4, lr: 3.00e-04 - train loss: 7.08e-01 - valid loss: 7.07e-01, valid CER: 3.17e-01, valid WER: 7.26e-01
+Epoch loaded: 4 - test loss: 7.05e-01, test CER: 2.37e-02, test WER: 6.91e-02

fluent-speech-commands/direct/train.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env/python3
+"""
+Recipe for "direct" (speech -> semantics) SLU with ASR-based transfer learning.
+
+We encode input waveforms into features using a model trained on LibriSpeech,
+then feed the features into a seq2seq model to map them to semantics.
+
+(Adapted from the LibriSpeech seq2seq ASR recipe written by Ju-Chieh Chou, Mirco Ravanelli, Abdel Heba, and Peter Plantinga.)
+
+Run using:
+> python train.py hparams/train.yaml
+
+Authors
+ * Loren Lugosch 2020
+ * Mirco Ravanelli 2020
+"""
+
+import sys
+import torch
+import speechbrain as sb
+import logging
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.utils.distributed import run_on_main
+
+logger = logging.getLogger(__name__)
+
+# Define training procedure
+
+
+class SLU(sb.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, tokens_bos_lens = batch.tokens_bos
+
+        # Add augmentation if specified
+        if stage == sb.Stage.TRAIN:
+            # Applying the augmentation pipeline
+            wavs_aug_tot = []
+            wavs_aug_tot.append(wavs)
+            for count, augment in enumerate(self.hparams.augment_pipeline):
+
+                # Apply augment
+                wavs_aug = augment(wavs, wav_lens)
+
+                # Managing speed change
+                if wavs_aug.shape[1] > wavs.shape[1]:
+                    wavs_aug = wavs_aug[:, 0 : wavs.shape[1]]
+                else:
+                    zero_sig = torch.zeros_like(wavs)
+                    zero_sig[:, 0 : wavs_aug.shape[1]] = wavs_aug
+                    wavs_aug = zero_sig
+
+                wavs_aug_tot.append(wavs_aug)
+
+            wavs = torch.cat(wavs_aug_tot, dim=0)
+            self.n_augment = len(wavs_aug_tot)
+            wav_lens = torch.cat([wav_lens] * self.n_augment)
+            tokens_bos = torch.cat([tokens_bos] * self.n_augment)
+
+        # ASR encoder forward pass
+        with torch.no_grad():
+            ASR_encoder_out = self.hparams.asr_model.encode_batch(
+                wavs.detach(), wav_lens
+            )
+
+        # SLU forward pass
+        encoder_out = self.hparams.slu_enc(ASR_encoder_out)
+        e_in = self.hparams.output_emb(tokens_bos)
+        h, _ = self.hparams.dec(e_in, encoder_out, wav_lens)
+
+        # Output layer for seq2seq log-probabilities
+        logits = self.hparams.seq_lin(h)
+        p_seq = self.hparams.log_softmax(logits)
+
+        # Compute outputs
+        if (
+            stage == sb.Stage.TRAIN
+            and self.batch_count % show_results_every != 0
+        ):
+            return p_seq, wav_lens
+        else:
+            p_tokens, scores = self.hparams.beam_searcher(encoder_out, wav_lens)
+            return p_seq, wav_lens, p_tokens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (NLL) given predictions and targets."""
+
+        if (
+            stage == sb.Stage.TRAIN
+            and self.batch_count % show_results_every != 0
+        ):
+            p_seq, wav_lens = predictions
+        else:
+            p_seq, wav_lens, predicted_tokens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        if hasattr(self.hparams, "env_corrupt") and stage == sb.Stage.TRAIN:
+            tokens_eos = torch.cat([tokens_eos, tokens_eos], dim=0)
+            tokens_eos_lens = torch.cat(
+                [tokens_eos_lens, tokens_eos_lens], dim=0
+            )
+
+        if stage == sb.Stage.TRAIN:
+            tokens_eos = torch.cat([tokens_eos] * self.n_augment, dim=0)
+            tokens_eos_lens = torch.cat(
+                [tokens_eos_lens] * self.n_augment, dim=0
+            )
+
+        loss_seq = self.hparams.seq_cost(
+            p_seq, tokens_eos, length=tokens_eos_lens
+        )
+
+        # (No ctc loss)
+        loss = loss_seq
+
+        if (stage != sb.Stage.TRAIN) or (
+            self.batch_count % show_results_every == 0
+        ):
+            # Decode token terms to words
+            predicted_semantics = [
+                tokenizer.decode_ids(utt_seq).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+
+            target_semantics = [wrd.split(" ") for wrd in batch.semantics]
+
+            for i in range(len(target_semantics)):
+                print(" ".join(predicted_semantics[i]).replace("|", ","))
+                print(" ".join(target_semantics[i]).replace("|", ","))
+                print("")
+
+            if stage != sb.Stage.TRAIN:
+                self.wer_metric.append(
+                    ids, predicted_semantics, target_semantics
+                )
+                self.cer_metric.append(
+                    ids, predicted_semantics, target_semantics
+                )
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        predictions = self.compute_forward(batch, sb.Stage.TRAIN)
+        loss = self.compute_objectives(predictions, batch, sb.Stage.TRAIN)
+        loss.backward()
+        if self.check_gradients(loss):
+            self.optimizer.step()
+        self.optimizer.zero_grad()
+        self.batch_count += 1
+        return loss.detach()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        self.batch_count = 0
+
+        if stage != sb.Stage.TRAIN:
+
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of a epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr, new_lr = self.hparams.lr_annealing(stage_stats["WER"])
+            sb.nnet.schedulers.update_learning_rate(self.optimizer, new_lr)
+            self.hparams.train_logger.log_stats(
+                stats_meta={"epoch": epoch, "lr": old_lr},
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_train"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(sort_key="duration")
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration", reverse=True
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_opts"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_valid"], replacements={"data_root": data_folder},
+    )
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["csv_test"], replacements={"data_root": data_folder},
+    )
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    tokenizer = hparams["tokenizer"]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        sig = sb.dataio.dataio.read_audio(wav)
+        return sig
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("semantics")
+    @sb.utils.data_pipeline.provides(
+        "semantics", "token_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(semantics):
+        yield semantics
+        tokens_list = tokenizer.encode_as_ids(semantics)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets,
+        ["id", "sig", "semantics", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data, tokenizer
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    show_results_every = 100  # plots results every N iterations
+
+    # If distributed_launch=True then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Dataset prep
+    from prepare import prepare_FSC  # noqa
+
+    # multi-gpu (ddp) save data preparation
+    run_on_main(
+        prepare_FSC,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["output_folder"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # here we create the datasets objects as well as tokenization and encoding
+    (train_set, valid_set, test_set, tokenizer,) = dataio_prepare(hparams)
+
+    # We download and pretrain the tokenizer
+    run_on_main(hparams["pretrainer"].collect_files)
+    hparams["pretrainer"].load_collected(device=run_opts["device"])
+
+    # Brain class initialization
+    slu_brain = SLU(
+        modules=hparams["modules"],
+        opt_class=hparams["opt_class"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # adding objects to trainer:
+    slu_brain.tokenizer = tokenizer
+
+    # Training
+    slu_brain.fit(
+        slu_brain.hparams.epoch_counter,
+        train_set,
+        valid_set,
+        train_loader_kwargs=hparams["dataloader_opts"],
+        valid_loader_kwargs=hparams["dataloader_opts"],
+    )
+
+    # Test
+    slu_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    slu_brain.evaluate(test_set, test_loader_kwargs=hparams["dataloader_opts"])

fluent-speech-commands/extra_requirements.txt

@@ -0,0 +1 @@
+pandas

fluent-speech-commands/prepare.py

@@ -0,0 +1,103 @@
+import os
+import logging
+from speechbrain.dataio.dataio import read_audio
+
+try:
+    import pandas as pd
+except ImportError:
+    err_msg = (
+        "The optional dependency pandas must be installed to run this recipe.\n"
+    )
+    err_msg += "Install using `pip install pandas`.\n"
+    raise ImportError(err_msg)
+
+logger = logging.getLogger(__name__)
+
+
+def prepare_FSC(data_folder, save_folder, skip_prep=False):
+    """
+    This function prepares the Fluent Speech Commands dataset.
+
+    data_folder : path to dataset.
+    save_folder: folder where the manifest files will be stored.
+    skip_prep: If True, skip data preparation
+
+    """
+    if skip_prep:
+        return
+
+    splits = [
+        "train",
+        "valid",
+        "test",
+    ]
+    ID_start = 0  # needed to have a unique ID for each audio
+    for split in splits:
+        new_filename = os.path.join(save_folder, split) + ".csv"
+        if os.path.exists(new_filename):
+            continue
+        logger.info("Preparing %s..." % new_filename)
+
+        ID = []
+        duration = []
+
+        wav = []
+        wav_format = []
+        wav_opts = []
+
+        spk_id = []
+        spk_id_format = []
+        spk_id_opts = []
+
+        semantics = []
+        semantics_format = []
+        semantics_opts = []
+
+        transcript = []
+        transcript_format = []
+        transcript_opts = []
+
+        df = pd.read_csv(os.path.join(data_folder, "data", split) + "_data.csv")
+        for i in range(len(df)):
+            ID.append(ID_start + i)
+            signal = read_audio(os.path.join(data_folder, df.path[i]))
+            duration.append(signal.shape[0] / 16000)
+
+            wav.append(os.path.join(data_folder, df.path[i]))
+            wav_format.append("wav")
+            wav_opts.append(None)
+
+            spk_id.append(df.speakerId[i])
+            spk_id_format.append("string")
+            spk_id_opts.append(None)
+
+            transcript_ = df.transcription[i]
+            transcript.append(transcript_)
+            transcript_format.append("string")
+            transcript_opts.append(None)
+
+            semantics_ = (
+                '{"action:" "'
+                + df.action[i]
+                + '"| "object": "'
+                + df.object[i]
+                + '"| "location": "'
+                + df.location[i]
+                + '"}'
+            )
+            semantics.append(semantics_)
+            semantics_format.append("string")
+            semantics_opts.append(None)
+
+        new_df = pd.DataFrame(
+            {
+                "ID": ID,
+                "duration": duration,
+                "wav": wav,
+                "spk_id": spk_id,
+                "semantics": semantics,
+                "transcript": transcript,
+            }
+        )
+        new_df.to_csv(new_filename, index=False)
+        ID_start += len(df)

pretrained_models/EncoderDecoderASR--5348169877143464308/asr.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/83e944252a91fe1d0883daa1e87077df4d64c35fffb45e22fff924faace4a59c.7fdf4aabd8400c69a6228ccc17c83b7a8ebf34c5d76f23497b7cf0d7a1baaea3

pretrained_models/EncoderDecoderASR--5348169877143464308/hyperparams.yaml

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/7aac72d39109ee19b4004d94239c2924caf33de6d85b0aff9296d844982210cb.d14310ea63844fb38520a592ea3a92e4f131b5f4683f8fa08e27b1e403c92293

pretrained_models/EncoderDecoderASR--5348169877143464308/lm.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/651df066b5d0b2efef7208f51df93d3a0a65bedc3a3a2500cd7b8faf064e631e.b438b9af3f549a23c4458bb066c11cd51dc1cfe9bfef30d3eb66b472e93b1e8c

pretrained_models/EncoderDecoderASR--5348169877143464308/normalizer.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/e733854cce680bcb58ce4b86bacb3cab5222880933b7b85ab17758aa5b10e9da.587fb748e80e719ed5721d5e0098c5feb2a901017135271ce2b2c6baea7e9f6e

pretrained_models/EncoderDecoderASR--5348169877143464308/tokenizer.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/f39208eba495042a59a8404b5703ca08a39a85e4d2bf707e197b90a3323f92ab.cd7af7ea8cfcfbf0f6dd61514c361972eb82b3b76f12b0e9ee0b371f36fdc078

pretrained_models/EndToEndSLU-7990244956535603082/hyperparams.yaml

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/a095f802a6283ecd636ffd0c0ec2d2dc335dcccfb395f5bc8d48fdb0ed34ca62.ca16cf2255d592246550b1dcfb9ac24800ec38cb8589cfd07e9db7558562037f

pretrained_models/EndToEndSLU-7990244956535603082/model.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/f01892eb014043257a527de1a0ebf610a17895a2b4c13d7e7e719c37231d08e5.d625fbcb8a2387e5d81fe6ff0d868125c7dcbc1b2245206ea152cccfb98a44fe

pretrained_models/EndToEndSLU-7990244956535603082/tokenizer.ckpt

@@ -0,0 +1 @@
+/root/.cache/huggingface/hub/2e5567fd31be3518b2a174a53d89d98df57247924ea50e69bbdb39cc4f8a76e5.8d38059f23fb577abadc9e131f1b67dd9662567eb032fdb8837e33a90feb47d4