```python
from transformers import DistilBertModel,AutoTokenizer
from transformers.models.distilbert.modeling_distilbert import *
import math
import torch
from torch import nn as nn

class PreTTRConfig(DistilBertConfig):
    join_layer_idx = 3

class PreTTR(DistilBertModel):
    '''
    PreTTR changes the distilbert model from huggingface to be able to split query and document until a set layer,
    we skipped compression present in the original

    from: Efficient Document Re-Ranking for Transformers by Precomputing Term Representations
          MacAvaney, et al. https://arxiv.org/abs/2004.14255
    '''
    config_class = PreTTRConfig

    def __init__(self, config):
        super().__init__(config)
        self.transformer = SplitTransformer(config)  # Encoder, we override the classes, but the names stay the same -> so it gets properly initialized
        self.embeddings = PosOffsetEmbeddings(config)  # Embeddings
        self._classification_layer = torch.nn.Linear(self.config.hidden_size, 1, bias=False)

        self.join_layer_idx = config.join_layer_idx

    def forward(
            self,
            query,
            document,
            use_fp16: bool = False) -> torch.Tensor:

        with torch.cuda.amp.autocast(enabled=use_fp16):

            query_input_ids = query["input_ids"]
            query_attention_mask = query["attention_mask"]

            document_input_ids = document["input_ids"][:, 1:]
            document_attention_mask = document["attention_mask"][:, 1:]

            query_embs = self.embeddings(query_input_ids)  # (bs, seq_length, dim)
            document_embs = self.embeddings(document_input_ids, query_input_ids.shape[-1])  # (bs, seq_length, dim)

            tfmr_output = self.transformer(
                query_embs=query_embs,
                query_mask=query_attention_mask,
                doc_embs=document_embs,
                doc_mask=document_attention_mask,
                join_layer_idx=self.join_layer_idx
            )
            hidden_state = tfmr_output[0]

            score = self._classification_layer(hidden_state[:, 0, :]).squeeze()

            return score

class PosOffsetEmbeddings(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.word_embeddings = nn.Embedding(config.vocab_size, config.dim, padding_idx=config.pad_token_id)
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.dim)
        if config.sinusoidal_pos_embds:
            create_sinusoidal_embeddings(
                n_pos=config.max_position_embeddings, dim=config.dim, out=self.position_embeddings.weight
            )

        self.LayerNorm = nn.LayerNorm(config.dim, eps=1e-12)
        self.dropout = nn.Dropout(config.dropout)

    def forward(self, input_ids, pos_offset=0):
        """
        Parameters
        ----------
        input_ids: torch.tensor(bs, max_seq_length)
            The token ids to embed.

        Outputs
        -------
        embeddings: torch.tensor(bs, max_seq_length, dim)
            The embedded tokens (plus position embeddings, no token_type embeddings)
        """
        seq_length = input_ids.size(1)
        position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device)  # (max_seq_length)
        position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + pos_offset  # (bs, max_seq_length)

        word_embeddings = self.word_embeddings(input_ids)  # (bs, max_seq_length, dim)
        position_embeddings = self.position_embeddings(position_ids)  # (bs, max_seq_length, dim)

        embeddings = word_embeddings + position_embeddings  # (bs, max_seq_length, dim)
        embeddings = self.LayerNorm(embeddings)  # (bs, max_seq_length, dim)
        embeddings = self.dropout(embeddings)  # (bs, max_seq_length, dim)
        return embeddings

class SplitTransformer(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.n_layers = config.n_layers

        layer = TransformerBlock(config)
        self.layer = nn.ModuleList([copy.deepcopy(layer) for _ in range(config.n_layers)])

    def forward(self, query_embs, query_mask, doc_embs, doc_mask, join_layer_idx, output_attentions=False, output_hidden_states=False):
        """
        Parameters
        ----------
        x: torch.tensor(bs, seq_length, dim)
            Input sequence embedded.
        attn_mask: torch.tensor(bs, seq_length)
            Attention mask on the sequence.

        Outputs
        -------
        hidden_state: torch.tensor(bs, seq_length, dim)
            Sequence of hiddens states in the last (top) layer
        all_hidden_states: Tuple[torch.tensor(bs, seq_length, dim)]
            Tuple of length n_layers with the hidden states from each layer.
            Optional: only if output_hidden_states=True
        all_attentions: Tuple[torch.tensor(bs, n_heads, seq_length, seq_length)]
            Tuple of length n_layers with the attention weights from each layer
            Optional: only if output_attentions=True
        """
        all_hidden_states = ()
        all_attentions = ()

        #
        # query / doc sep.
        #
        hidden_state_q = query_embs
        hidden_state_d = doc_embs
        for layer_module in self.layer[:join_layer_idx]:

            layer_outputs_q = layer_module(
                x=hidden_state_q, attn_mask=query_mask, head_mask=None, output_attentions=output_attentions
            )
            hidden_state_q = layer_outputs_q[-1]

            layer_outputs_d = layer_module(
                x=hidden_state_d, attn_mask=doc_mask, head_mask=None, output_attentions=output_attentions
            )
            hidden_state_d = layer_outputs_d[-1]

        #
        # combine
        #
        x = torch.cat([hidden_state_q, hidden_state_d], dim=1)
        attn_mask = torch.cat([query_mask, doc_mask], dim=1)

        #
        # combined
        #
        hidden_state = x
        for layer_module in self.layer[join_layer_idx:]:
            layer_outputs = layer_module(
                x=hidden_state, attn_mask=attn_mask, head_mask=None, output_attentions=output_attentions
            )
            hidden_state = layer_outputs[-1]

        # Add last layer
        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_state,)

        outputs = (hidden_state,)
        if output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if output_attentions:
            outputs = outputs + (all_attentions,)
        return outputs  # last-layer hidden state, (all hidden states), (all attentions)

#
# init the model & tokenizer (using the distilbert tokenizer)
#
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased") # honestly not sure if that is the best way to go, but it works :)
model = PreTTR.from_pretrained("sebastian-hofstaetter/prettr-distilbert-split_at_3-margin_mse-T2-msmarco")
```


## Effectiveness on MSMARCO Passage


We trained our model on the MSMARCO standard ("small"-400K query) training triples with knowledge distillation with a batch size of 32 on a single consumer-grade GPU (11GB memory).


For re-ranking we used the top-1000 BM25 results.


### MSMARCO-DEV


Here, we use the larger 49K query DEV set (same range as the smaller 7K DEV set, minimal changes possible)


|                                  | MRR@10 | NDCG@10 |
|----------------------------------|--------|---------|
| BM25                             | .194   | .241    |
| **Margin-MSE PreTTR** (Re-ranking) | .386   | .447   |


For more metrics, baselines, info and analysis, please see the paper: https://arxiv.org/abs/2010.02666


## Limitations & Bias


- The model inherits social biases from both DistilBERT and MSMARCO.
- The model is only trained on relatively short passages of MSMARCO (avg. 60 words length), so it might struggle with longer text.


## Citation


If you use our model checkpoint please cite our work as:


```
@misc{hofstaetter2020_crossarchitecture_kd,
      title={Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation},
      author={Sebastian Hofst{\"a}tter and Sophia Althammer and Michael Schr{\"o}der and Mete Sertkan and Allan Hanbury},
      year={2020},
      eprint={2010.02666},
      archivePrefix={arXiv},
      primaryClass={cs.IR}
}
```


model name: prettr-distilbert-split_at_3-margin_mse-T2-msmarco


model author: sebastian-hofstaetter


tags: pytorch, distilbert, en, dataset:ms_marco, arxiv:2004.14255, arxiv:2010.02666, transformers, knowledge-distillation, infinity_compatible


language: "en"
tags:


- dpr
- dense-passage-retrieval
- knowledge-distillation
datasets:
- ms_marco


# Margin-MSE Trained ColBERT


We provide a retrieval trained DistilBert-based ColBERT model (https://arxiv.org/pdf/2004.12832.pdf). Our model is trained with Margin-MSE using a 3 teacher BERT_Cat (concatenated BERT scoring) ensemble on MSMARCO-Passage.


This instance can be used to **re-rank a candidate set** or **directly for a vector index based dense retrieval**. The architecure is a 6-layer DistilBERT, with an additional single linear layer at the end.


If you want to know more about our simple, yet effective knowledge distillation method for efficient information retrieval models for a variety of student architectures that is used for this model instance check out our paper: https://arxiv.org/abs/2010.02666 🎉


For more information, training data, source code, and a minimal usage example please visit: https://github.com/sebastian-hofstaetter/neural-ranking-kd


## Configuration


- fp16 trained, so fp16 inference shouldn't be a problem
- We use no compression: 768 dim output vectors (better suited for re-ranking, or storage for smaller collections, MSMARCO gets to ~1TB vector storage with fp16 ... ups)
- Query [MASK] augmention = 8x regardless of batch-size (needs to be added before the model, see the usage example in GitHub repo for more)


## Model Code


```python
from transformers import AutoTokenizer,AutoModel, PreTrainedModel,PretrainedConfig
from typing import Dict
import torch

class ColBERTConfig(PretrainedConfig):
    model_type = "ColBERT"
    bert_model: str
    compression_dim: int = 768
    dropout: float = 0.0
    return_vecs: bool = False
    trainable: bool = True

class ColBERT(PreTrainedModel):
    """
    ColBERT model from: https://arxiv.org/pdf/2004.12832.pdf
    We use a dot-product instead of cosine per term (slightly better)
    """
    config_class = ColBERTConfig
    base_model_prefix = "bert_model"

    def __init__(self,
                 cfg) -> None:
        super().__init__(cfg)

        self.bert_model = AutoModel.from_pretrained(cfg.bert_model)

        for p in self.bert_model.parameters():
            p.requires_grad = cfg.trainable

        self.compressor = torch.nn.Linear(self.bert_model.config.hidden_size, cfg.compression_dim)

    def forward(self,
                query: Dict[str, torch.LongTensor],
                document: Dict[str, torch.LongTensor]):

        query_vecs = self.forward_representation(query)
        document_vecs = self.forward_representation(document)

        score = self.forward_aggregation(query_vecs,document_vecs,query["attention_mask"],document["attention_mask"])
        return score

    def forward_representation(self,
                               tokens,
                               sequence_type=None) -> torch.Tensor:

        vecs = self.bert_model(**tokens)[0] # assuming a distilbert model here
        vecs = self.compressor(vecs)

        # if encoding only, zero-out the mask values so we can compress storage
        if sequence_type == "doc_encode" or sequence_type == "query_encode":
            vecs = vecs * tokens["tokens"]["mask"].unsqueeze(-1)

        return vecs

    def forward_aggregation(self,query_vecs, document_vecs,query_mask,document_mask):

        # create initial term-x-term scores (dot-product)
        score = torch.bmm(query_vecs, document_vecs.transpose(2,1))

        # mask out padding on the doc dimension (mask by -1000, because max should not select those, setting it to 0 might select them)
        exp_mask = document_mask.bool().unsqueeze(1).expand(-1,score.shape[1],-1)
        score[~exp_mask] = - 10000

        # max pooling over document dimension
        score = score.max(-1).values

        # mask out paddding query values
        score[~(query_mask.bool())] = 0

        # sum over query values
        score = score.sum(-1)

        return score

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased") # honestly not sure if that is the best way to go, but it works :)
model = ColBERT.from_pretrained("sebastian-hofstaetter/colbert-distilbert-margin_mse-T2-msmarco")
```


## Effectiveness on MSMARCO Passage & TREC Deep Learning '19


We trained our model on the MSMARCO standard ("small"-400K query) training triples with knowledge distillation with a batch size of 32 on a single consumer-grade GPU (11GB memory).


For re-ranking we used the top-1000 BM25 results.


### MSMARCO-DEV


Here, we use the larger 49K query DEV set (same range as the smaller 7K DEV set, minimal changes possible)


|                                  | MRR@10 | NDCG@10 |
|----------------------------------|--------|---------|
| BM25                             | .194   | .241    |
| **Margin-MSE ColBERT** (Re-ranking) | .375   | .436   |


### TREC-DL'19


For MRR we use the recommended binarization point of the graded relevance of 2. This might skew the results when compared to other binarization point numbers.


|                                  | MRR@10 | NDCG@10 |
|----------------------------------|--------|---------|
| BM25                             | .689   | .501    |
| **Margin-MSE ColBERT** (Re-ranking) | .878   | .744    |


For more metrics, baselines, info and analysis, please see the paper: https://arxiv.org/abs/2010.02666


## Limitations & Bias


- The model inherits social biases from both DistilBERT and MSMARCO.
- The model is only trained on relatively short passages of MSMARCO (avg. 60 words length), so it might struggle with longer text.


## Citation


If you use our model checkpoint please cite our work as:


```
@misc{hofstaetter2020_crossarchitecture_kd,
      title={Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation},
      author={Sebastian Hofst{\"a}tter and Sophia Althammer and Michael Schr{\"o}der and Mete Sertkan and Allan Hanbury},
      year={2020},
      eprint={2010.02666},
      archivePrefix={arXiv},
      primaryClass={cs.IR}
}
```


model name: colbert-distilbert-margin_mse-T2-msmarco


model author: sebastian-hofstaetter


tags: pytorch, ColBERT, en, dataset:ms_marco, arxiv:2004.12832, arxiv:2010.02666, transformers, dpr, dense-passage-retrieval, knowledge-distillation


language: "en"
tags:


- dpr
- dense-passage-retrieval
- knowledge-distillation
datasets:
- ms_marco


# DistilBert for Dense Passage Retrieval trained with Balanced Topic Aware Sampling (TAS-B)


We provide a retrieval trained DistilBert-based model (we call the *dual-encoder then dot-product scoring* architecture BERT_Dot) trained with Balanced Topic Aware Sampling on MSMARCO-Passage.


This instance was trained with a batch size of 256 and can be used to **re-rank a candidate set** or **directly for a vector index based dense retrieval**. The architecture is a 6-layer DistilBERT, without architecture additions or modifications (we only change the weights during training) - to receive a query/passage representation we pool the CLS vector. We use the same BERT layers for both query and passage encoding (yields better results, and lowers memory requirements).


If you want to know more about our efficient (can be done on a single consumer GPU in 48 hours) batch composition procedure and dual supervision for dense retrieval training, check out our paper: https://arxiv.org/abs/2104.06967 🎉


For more information and a minimal usage example please visit: https://github.com/sebastian-hofstaetter/tas-balanced-dense-retrieval


## Effectiveness on MSMARCO Passage & TREC-DL'19


We trained our model on the MSMARCO standard ("small"-400K query) training triples re-sampled with our TAS-B method. As teacher models we used the BERT_CAT pairwise scores as well as the ColBERT model for in-batch-negative signals published here: https://github.com/sebastian-hofstaetter/neural-ranking-kd


### MSMARCO-DEV (7K)


|                                  | MRR@10 | NDCG@10 | Recall@1K                   |
|----------------------------------|--------|---------|-----------------------------|
| BM25                             | .194   | .241    | .857  |
| **TAS-B BERT_Dot** (Retrieval)   | .347   | .410    | .978                        |


### TREC-DL'19


For MRR and Recall we use the recommended binarization point of the graded relevance of 2. This might skew the results when compared to other binarization point numbers.


|                                  | MRR@10 | NDCG@10 | Recall@1K                   |
|----------------------------------|--------|---------|-----------------------------|
| BM25                             | .689   | .501    | .739  |
| **TAS-B BERT_Dot** (Retrieval)   | .883   | .717    | .843                       |


### TREC-DL'20


|                                  | MRR@10 | NDCG@10 | Recall@1K                   |
|----------------------------------|--------|---------|-----------------------------|
| BM25                             | .649   | .475    | .806  |
| **TAS-B BERT_Dot** (Retrieval)   | .843   | .686    | .875                        |


For more baselines, info and analysis, please see the paper: https://arxiv.org/abs/2104.06967


## Limitations & Bias


- The model inherits social biases from both DistilBERT and MSMARCO.
- The model is only trained on relatively short passages of MSMARCO (avg. 60 words length), so it might struggle with longer text.


## Citation


If you use our model checkpoint please cite our work as:


```
@inproceedings{Hofstaetter2021_tasb_dense_retrieval,
 author = {Sebastian Hofst{\"a}tter and Sheng-Chieh Lin and Jheng-Hong Yang and Jimmy Lin and Allan Hanbury},
 title = {{Efficiently Teaching an Effective Dense Retriever with Balanced Topic Aware Sampling}},
 booktitle = {Proc. of SIGIR},
 year = {2021},
}
```


model name: distilbert-dot-tas_b-b256-msmarco


model author: sebastian-hofstaetter


task: feature-extraction


tags: pytorch, distilbert, feature-extraction, en, dataset:ms_marco, arxiv:2104.06967, transformers, dpr, dense-passage-retrieval, knowledge-distillation, infinity_compatible


language: "en"
tags:


- dpr
- dense-passage-retrieval
- knowledge-distillation
datasets:
- ms_marco


# Margin-MSE Trained DistilBert for Dense Passage Retrieval


We provide a retrieval trained DistilBert-based model (we call the architecture BERT_Dot). Our model is trained with Margin-MSE using a 3 teacher BERT_Cat (concatenated BERT scoring) ensemble on MSMARCO-Passage.


This instance can be used to **re-rank a candidate set** or **directly for a vector index based dense retrieval**. The architecture is a 6-layer DistilBERT, without architecture additions or modifications (we only change the weights during training) - to receive a query/passage representation we pool the CLS vector. We use the same BERT layers for both query and passage encoding (yields better results, and lowers memory requirements).


If you want to know more about our simple, yet effective knowledge distillation method for efficient information retrieval models for a variety of student architectures that is used for this model instance check out our paper: https://arxiv.org/abs/2010.02666 🎉


For more information, training data, source code, and a minimal usage example please visit: https://github.com/sebastian-hofstaetter/neural-ranking-kd


## Effectiveness on MSMARCO Passage & TREC-DL'19


We trained our model on the MSMARCO standard ("small"-400K query) training triples with knowledge distillation with a batch size of 32 on a single consumer-grade GPU (11GB memory).


For re-ranking we used the top-1000 BM25 results.


### MSMARCO-DEV


|                                  | MRR@10 | NDCG@10 | Recall@1K                   |
|----------------------------------|--------|---------|-----------------------------|
| BM25                             | .194   | .241    | .868  |
| **Margin-MSE BERT_Dot** (Re-ranking) | .332   | .391    | .868 (from BM25 candidates) |
| **Margin-MSE BERT_Dot** (Retrieval)  | .323   | .381    | .957                        |


### TREC-DL'19


For MRR and Recall we use the recommended binarization point of the graded relevance of 2. This might skew the results when compared to other binarization point numbers.


|                                  | MRR@10 | NDCG@10 | Recall@1K                   |
|----------------------------------|--------|---------|-----------------------------|
| BM25                             | .689   | .501    | .739  |
| **Margin-MSE BERT_Dot** (Re-ranking) | .862   | .712    | .739 (from BM25 candidates) |
| **Margin-MSE BERT_Dot** (Retrieval)  | .868   | .697    | .769                        |


For more baselines, info and analysis, please see the paper: https://arxiv.org/abs/2010.02666


## Limitations & Bias


- The model inherits social biases from both DistilBERT and MSMARCO.
- The model is only trained on relatively short passages of MSMARCO (avg. 60 words length), so it might struggle with longer text.


## Citation


If you use our model checkpoint please cite our work as:


```
@misc{hofstaetter2020_crossarchitecture_kd,
      title={Improving Efficient Neural Ranking Models with Cross-Architecture Knowledge Distillation},
      author={Sebastian Hofst{\"a}tter and Sophia Althammer and Michael Schr{\"o}der and Mete Sertkan and Allan Hanbury},
      year={2020},
      eprint={2010.02666},
      archivePrefix={arXiv},
      primaryClass={cs.IR}
}
```


model name: distilbert-dot-margin_mse-T2-msmarco


model author: sebastian-hofstaetter


task: feature-extraction


tags: pytorch, distilbert, feature-extraction, en, dataset:ms_marco, arxiv:2010.02666, transformers, dpr, dense-passage-retrieval, knowledge-distillation, infinity_compatible


language:


- jpn
- ain
tags:
- translation


mt5-small-ain-jpn-mt is a machine translation model pretrained with [Google's mT5-small](https://huggingface.co/google/mt5-small) and fine-tuned on bilingual datasets crawled from the Web. It translates Ainu language to Japanese.


model name: mt5-small-ain-jpn-mt


model author: gojiteji


task: translation


tags: pytorch, mt5, text2text-generation, jpn, ain, transformers, translation, autonlp_compatible


- ain
- ja
tags:
- translation


Byt5-small-ain-jpn-mt is a machine translation model pretrained with [Google's ByT5-small](https://huggingface.co/google/byt5-small) and fine-tuned on bilingual datasets crawled from the Web. It translates Ainu language to Japanese.


model name: byt5-small-ain-jpn-mt


tags: pytorch, t5, text2text-generation, ain, ja, transformers, translation, autonlp_compatible


mt5-small-jpn-ain-mt is a machine translation model pretrained with [Google's mT5-small](https://huggingface.co/google/mt5-small) and fine-tuned on bilingual datasets crawled from the Web. It translates Japanese to Ainu language.


model name: mt5-small-jpn-ain-mt


Byt5-small-jpn-ain-mt is a machine translation model pretrained with [Google's ByT5-small](https://huggingface.co/google/byt5-small) and fine-tuned on bilingual datasets crawled from the Web. It translates Japanese to Ainu language.


model name: byt5-small-jpn-ain-mt


tags: pytorch, t5, text2text-generation, jpn, ain, transformers, translation, autonlp_compatible


tags:


- conversational


#Saitama DialoGPT model


model name: DialoGPT-small-saitama


model author: Daivakai


task: conversational


tags: pytorch, gpt2, text-generation, transformers, conversational


BERT for Vietnamese Law


Apply for Task 1: Legal Document Retrieval on ALQAC 2021 dataset


The model achieved 0.80 on the leaderboard(1st place score is 0.88).


We use vibert4news as based model and fine-tune on our own Vietnamese law dataset.


We use word sentencepiece, use basic bert tokenization and same config with bert base with lowercase = False.


model name: AimeLaw


model author: ductuan024


task: fill-mask


tags: pytorch, ibert, fill-mask, transformers, autonlp_compatible


license: afl-3.0
tags:


- es
- bert
pipeline_tag: text-classification
widget:
- text: "Mi nombre es Omar"
exdample_title: "Example 1"
- text: "Otra prueba"
example_title: "Test"


# Test


model name: test


model author: hackertec9


task: text-classification


tags: license:afl-3.0, es, bert, text-classification


BERT Language Model Further Pre-trained on Persian Poetry


model name: BERT-Persian-Poetry


model author: mitra-mir


tags: pytorch, tf, jax, bert, fill-mask, transformers, autonlp_compatible, infinity_compatible


A Transformer-based Persian Language Model Further Pretrained on Persian Poetry


ALBERT was first introduced by [Hooshvare](https://huggingface.co/HooshvareLab/albert-fa-zwnj-base-v2?text=%D8%B2+%D8%A2%D9%86+%D8%AF%D8%B1%D8%AF%D8%B4+%5BMASK%5D+%D9%85%DB%8C+%D8%B3%D9%88%D8%AE%D8%AA+%D8%AF%D8%B1+%D8%A8%D8%B1) with 30,000 vocabulary size as lite BERT for self-supervised learning of language representations for the Persian language. Here we wanted to utilize its capabilities by pretraining it on a large corpse of Persian poetry. This model has been post-trained on 80 percent of poetry verses of the Persian poetry dataset - Ganjoor- and has been evaluated on the other 20 percent.


model name: ALBERT-Persian-Poetry


tags: pytorch, tf, albert, fill-mask, transformers, autonlp_compatible, infinity_compatible


tags:


- conversational


# My Awesome Model


model name: DialoGPT-small-joshua


model author: ClydeWasTaken


task: conversational


tags: conversational


# SpongeBob DialoGPT Model


model name: DialoGPT-spongebob-small


model author: mutamuta


# Rick DialoGPT Model


model name: DialoGPT-small-rick


# Harry Potter DialogGPT Model


model name: DialogGPT-small-HPBot


model author: setiadia


## Model description


This model is a fine-tuned version of the [DistilBERT model](https://huggingface.co/transformers/model_doc/distilbert.html) to classify toxic comments.


## How to use


You can use the model with the following code.


```python
from transformers import AutoModelForSequenceClassification, AutoTokenizer, TextClassificationPipeline

model_path = "martin-ha/toxic-comment-model"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForSequenceClassification.from_pretrained(model_path)

pipeline =  TextClassificationPipeline(model=model, tokenizer=tokenizer)
print(pipeline('This is a test text.'))
```


## Limitations and Bias


This model is intended to use for classify toxic online classifications. However, one limitation of the model is that it performs poorly for some comments that mention a specific identity subgroup, like Muslim. The following table shows a evaluation score for different identity group. You can learn the specific meaning of this metrics [here](https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/overview/evaluation). But basically, those metrics shows how well a model performs for a specific group. The larger the number, the better.


| **subgroup**                  | **subgroup_size** | **subgroup_auc** | **bpsn_auc** | **bnsp_auc** |
| ----------------------------- | ----------------- | ---------------- | ------------ | ------------ |
| muslim                        | 108               | 0.689            | 0.811        | 0.88         |
| jewish                        | 40                | 0.749            | 0.86         | 0.825        |
| homosexual_gay_or_lesbian     | 56                | 0.795            | 0.706        | 0.972        |
| black                         | 84                | 0.866            | 0.758        | 0.975        |
| white                         | 112               | 0.876            | 0.784        | 0.97         |
| female                        | 306               | 0.898            | 0.887        | 0.948        |
| christian                     | 231               | 0.904            | 0.917        | 0.93         |
| male                          | 225               | 0.922            | 0.862        | 0.967        |
| psychiatric_or_mental_illness | 26                | 0.924            | 0.907        | 0.95         |


The table above shows that the model performs poorly for the muslim and jewish group. In fact, you pass the sentence "Muslims are people who follow or practice Islam, an Abrahamic monotheistic religion." Into the model, the model will classify it as toxic. Be mindful for this type of potential bias.


## Training data


The training data comes this [Kaggle competition](https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/data). We use 10% of the `train.csv` data to train the model.


## Training procedure


You can see [this documentation and codes](https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/data) for how we train the model. It takes about 3 hours in a P-100 GPU.


## Evaluation results


The model achieves 94% accuracy and 0.59 f1-score in a 10000 rows held-out test set.


model name: toxic-comment-model


model author: martin-ha


task: text-classification


tags: pytorch, distilbert, text-classification, transformers, infinity_compatible


tags:


- conversational


# Rick and Morty DialoGPT Model


model name: DialoGPT-small-Rick


model author: Oji


task: conversational


tags: pytorch, gpt2, text-generation, transformers, conversational


# Harry Potter DialoGPT Model


model name: DialoGPT-small-harry


model author: cutiebunny639


license: mit
tags:


- generated_from_trainer
model-index:
- name: BERiTmodel3
results: []


<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->


# BERiTmodel3


This model is a fine-tuned version of [roberta-base](https://huggingface.co/roberta-base) on an unknown dataset.
It achieves the following results on the evaluation set:


- Loss: 3.4605


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


The following hyperparameters were used during training:


- learning_rate: 5e-05
- train_batch_size: 8
- eval_batch_size: 8
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: cosine
- lr_scheduler_warmup_steps: 280
- num_epochs: 10


### Training results


| Training Loss | Epoch | Step  | Validation Loss |
|:-------------:|:-----:|:-----:|:---------------:|
| 4.1329        | 1.0   | 2854  | 3.9788          |
| 3.8517        | 2.0   | 5708  | 4.0719          |
| 3.6145        | 3.0   | 8562  | 3.6353          |
| 3.4563        | 4.0   | 11416 | 3.8803          |
| 3.364         | 5.0   | 14270 | 3.6327          |
| 3.2525        | 6.0   | 17124 | 3.2436          |
| 3.1732        | 7.0   | 19978 | 3.3924          |
| 3.1542        | 8.0   | 22832 | 3.2765          |
| 3.0772        | 9.0   | 25686 | 3.2873          |
| 3.0236        | 10.0  | 28540 | 3.4605          |


### Framework versions


- Transformers 4.15.0
- Pytorch 1.10.0+cu111
- Datasets 1.17.0
- Tokenizers 0.10.3


model name: BERiTmodel3


model author: gngpostalsrvc


task: fill-mask


tags: pytorch, tensorboard, roberta, fill-mask, transformers, license:mit, generated_from_trainer, model-index, autonlp_compatible, infinity_compatible


license: mit
tags:


- generated_from_trainer
model-index:
- name: BERiTmodel2
results: []


# BERiTmodel2


- Loss: 3.1508


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


### Training results


| Training Loss | Epoch | Step  | Validation Loss |
|:-------------:|:-----:|:-----:|:---------------:|
| 3.1924        | 1.0   | 2854  | 3.4329          |
| 3.0936        | 2.0   | 5708  | 3.5036          |
| 2.9998        | 3.0   | 8562  | 3.1906          |
| 2.9064        | 4.0   | 11416 | 3.4867          |
| 2.8493        | 5.0   | 14270 | 3.2027          |
| 2.7538        | 6.0   | 17124 | 2.9772          |
| 2.7273        | 7.0   | 19978 | 2.9950          |
| 2.7399        | 8.0   | 22832 | 2.9690          |
| 2.67          | 9.0   | 25686 | 3.0311          |
| 2.6388        | 10.0  | 28540 | 3.1508          |


### Framework versions


- Transformers 4.14.1
- Pytorch 1.10.0+cu111
- Datasets 1.17.0
- Tokenizers 0.10.3


model name: BERiTmodel2


model author: gngpostalsrvc


task: fill-mask


license: mit
tags:


- generated_from_trainer
model-index:
- name: BERiTmodel
results: []


# BERiTmodel


- Loss: 3.4360


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


### Training results


| Training Loss | Epoch | Step  | Validation Loss |
|:-------------:|:-----:|:-----:|:---------------:|
| 4.1331        | 1.0   | 2854  | 4.0569          |
| 3.8434        | 2.0   | 5708  | 4.0501          |
| 3.6089        | 3.0   | 8562  | 3.5999          |
| 3.4504        | 4.0   | 11416 | 3.9287          |
| 3.3539        | 5.0   | 14270 | 3.6256          |
| 3.2302        | 6.0   | 17124 | 3.2316          |
| 3.1616        | 7.0   | 19978 | 3.3701          |
| 3.1431        | 8.0   | 22832 | 3.2708          |
| 3.0646        | 9.0   | 25686 | 3.3050          |
| 3.0096        | 10.0  | 28540 | 3.4360          |


### Framework versions


model name: BERiTmodel


## license: eupl-1.1


## BERT model van het project Explainable AI


model name: BERT_responsible_AI


model author: Mustang


task: text-classification


tags: pytorch, bert, text-classification, transformers, license:eupl-1.1, infinity_compatible


tags:


- conversational


# Harry Potter DialoGPT Model


model name: DialoGPT-medium-harrypotterall


model author: Aruden


task: conversational


tags: pytorch, gpt2, text-generation, transformers, conversational


# RoBERTa-minuscule


A ridiculously small model for testing purposes.


model name: roberta-minuscule


model author: lgrobol


task: fill-mask


tags: pytorch, roberta, fill-mask, transformers, autonlp_compatible, infinity_compatible


language: multilingual
tags:


- electra
- testing
- minuscule
license: "cc0-1.0"


# ELECTRA-minuscule-generator


A ridiculously small ELECTRA generator model for testing purposes.


**THIS MODEL HAS NOT BEEN TRAINED, DO NOT EXPECT ANYThING OF IT.**


model name: electra-minuscule-generator


tags: pytorch, electra, fill-mask, multilingual, transformers, testing, minuscule, license:cc0-1.0, autonlp_compatible, infinity_compatible


# FlauBERT-minuscule


model name: flaubert-minuscule


tags: pytorch, flaubert, fill-mask, transformers, autonlp_compatible


language: multilingual
thumbnail: "url to a thumbnail used in social sharing"
tags:


# ELECTRA-minuscule-discriminator


A ridiculously small ELECTRA discriminator model for testing purposes.


model name: electra-minuscule-discriminator


model author: lgrobol


task: token-classification


tags: pytorch, electra, token-classification, multilingual, transformers, testing, minuscule, license:cc0-1.0, autonlp_compatible, infinity_compatible


language: ar
widget:


- text: " جاب ليا [MASK] ."
- text: "مشيت نجيب[MASK] فالفرماسيان ."


AIOX Lab and  SI2M Lab INSEA have joined forces to offer researchers, industrialists and the NLP (Natural Language Processing) community the first intelligent Open Source system that understands Moroccan dialectal language "Darija".


**DarijaBERT** is the first BERT model for the Moroccan Arabic dialect called “Darija”. It is based on the same architecture as BERT-base, but without the Next Sentence Prediction (NSP) objective. This model was trained on a total of ~3 Million sequences of Darija dialect representing 691MB of text or a total of ~100M tokens.


The model was trained on a dataset issued from three different sources:


* Stories written in Darija scrapped from a dedicated website
* Youtube comments from 40 different Moroccan channels
* Tweets crawled based on a list of Darija keywords.


More details about DarijaBert are available in the dedicated GitHub [repository](https://github.com/AIOXLABS/DBert)


**Loading the model**


The model can be loaded directly using the Huggingface library:


```python
from transformers import AutoTokenizer, AutoModel
DarijaBERT_tokenizer = AutoTokenizer.from_pretrained("Kamel/DarijaBERT")
DarijaBert_model = AutoModel.from_pretrained("Kamel/DarijaBERT")
```


**Acknowledgments**


We gratefully acknowledge Google’s TensorFlow Research Cloud (TRC) program for providing us with free Cloud TPUs.


model name: DarijaBERT


model author: Kamel


task: fill-mask


tags: pytorch, bert, fill-mask, ar, transformers, autonlp_compatible, infinity_compatible


- text: " Mchit njib [MASK] ."
- text: " Yak nta li [MASK] lih dik lhedra."
- text: " Ach [MASK] daba."
- text: " Lmghrib ajmal [MASK] fl3alam."


AIOX Lab and  SI2M Lab INSEA have joined forces to offer researchers, industrialists and the NLP (Natural Language Processing) community the first intelligent Open Source system that understands Moroccan dialectal language "Darija".
**DarijaBERT** is the first BERT model for the Moroccan Arabic dialect called “Darija”. It is based on the same architecture as BERT-base, but without the Next Sentence Prediction (NSP) objective. This model is the Arabizi specific version of DarijaBERT and it was trained on a total of ~4.6 Million sequences of Darija dialect written in Latin letters.


The model was trained on a dataset issued from Youtube comments.


More details about DarijaBert are available in the dedicated GitHub [repository](https://github.com/AIOXLABS/DBert)
**Loading the model**
The model can be loaded directly using the Huggingface library:


```python
from transformers import AutoTokenizer, AutoModel
DarijaBERT_tokenizer = AutoTokenizer.from_pretrained("Kamel/DarijaBERT-arabizi")
DarijaBert_model = AutoModel.from_pretrained("Kamel/DarijaBERT-arabizi")
```


**Acknowledgments**
We gratefully acknowledge Google’s TensorFlow Research Cloud (TRC) program for providing us with free Cloud TPUs.


**Warning**
This model being trained on texts from social networks, it can unfortunately generate toxic outputs reflecting part of the learned data


model name: DarijaBERT-arabizi


language: nl
datasets:


- common_voice
metrics:
- wer
tags:
- audio
- automatic-speech-recognition
- speech
- xlsr-fine-tuning-week
license: apache-2.0
model-index:
- name: simonsr wav2vec2-large-xlsr-dutch
results:
  - task:
name: Speech Recognition
type: automatic-speech-recognition
dataset:
name: Common Voice nl
type: common_voice
args: nl
metrics:
    - name: Test WER
type: wer
value: 38.74


# Wav2Vec2-Large-XLSR-53-Dutch


Fine-tuned [facebook/wav2vec2-large-xlsr-53](https://huggingface.co/facebook/wav2vec2-large-xlsr-53) on Dutch using the [Common Voice](https://huggingface.co/datasets/common_voice)


When using this model, make sure that your speech input is sampled at 16kHz.


## Usage


The model can be used directly (without a language model) as follows:


```python
import torch
import torchaudio
from datasets import load_dataset
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor

test_dataset = load_dataset("common_voice", "nl", split="test[:2%]")

processor = Wav2Vec2Processor.from_pretrained("simonsr/wav2vec2-large-xlsr-dutch")
model = Wav2Vec2ForCTC.from_pretrained("simonsr/wav2vec2-large-xlsr-dutch")

resampler = torchaudio.transforms.Resample(48_000, 16_000)

# Preprocessing the datasets.
# We need to read the audio files as arrays
def speech_file_to_array_fn(batch):
    speech_array, sampling_rate = torchaudio.load(batch["path"])
    batch["speech"] = resampler(speech_array).squeeze().numpy()
    return batch

test_dataset = test_dataset.map(speech_file_to_array_fn)
inputs = processor(test_dataset["speech"][:2], sampling_rate=16_000, return_tensors="pt", padding=True)

with torch.no_grad():
    logits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits

predicted_ids = torch.argmax(logits, dim=-1)

print("Prediction:", processor.batch_decode(predicted_ids))
print("Reference:", test_dataset["sentence"][:2])
```


## Evaluation


The model can be evaluated as follows on the Dutch test data of Common Voice.


```python
import torch
import torchaudio
from datasets import load_dataset, load_metric
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
import unidecode
import re

test_dataset = load_dataset("common_voice", "nl", split="test")
wer = load_metric("wer")

processor = Wav2Vec2Processor.from_pretrained("{model_id}") #TODO: replace {model_id} with your model id. The model id consists of {your_username}/{your_modelname}, *e.g.* `elgeish/wav2vec2-large-xlsr-53-arabic`
model = Wav2Vec2ForCTC.from_pretrained("{model_id}") #TODO: replace {model_id} with your model id. The model id consists of {your_username}/{your_modelname}, *e.g.* `elgeish/wav2vec2-large-xlsr-53-arabic`
model.to("cuda")

chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\(\)\=\´\–\&\…\—\’]'
resampler = torchaudio.transforms.Resample(48_000, 16_000)

# Preprocessing the datasets.
# We need to read the aduio files as arrays
def speech_file_to_array_fn(batch):
    batch["sentence"] = unidecode.unidecode(batch["sentence"])
    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
    speech_array, sampling_rate = torchaudio.load(batch["path"])
    batch["speech"] = resampler(speech_array).squeeze().numpy()
    return batch

test_dataset = test_dataset.map(speech_file_to_array_fn)

# Preprocessing the datasets.
# We need to read the aduio files as arrays
def evaluate(batch):
    inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)

    with torch.no_grad():
        logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits

    pred_ids = torch.argmax(logits, dim=-1)

    batch["pred_strings"] = processor.batch_decode(pred_ids)
     return batch

result = test_dataset.map(evaluate, batched=True, batch_size=8)

print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
```


**Test Result**: 38.74 %


## Training


The Common Voice `train`, `validation`, and ... datasets were used for training.


The script used for training can be found [here](...) # TODO: fill in a link to your training script here. If you trained your model in a colab, simply fill in the link here. If you trained the model locally, it would be great if you could upload the training script on github and paste the link here.


model name: wav2vec2-large-xlsr-dutch


model author: simonsr


task: automatic-speech-recognition


tags: pytorch, wav2vec2, automatic-speech-recognition, nl, dataset:common_voice, transformers, audio, speech, xlsr-fine-tuning-week, license:apache-2.0, model-index


license: apache-2.0
tags:


- generated_from_trainer
model-index:
name: wav2vec2-base-myst-demo-colab


<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->


# wav2vec2-base-myst-demo-colab


This model is a fine-tuned version of [facebook/wav2vec2-base-960h](https://huggingface.co/facebook/wav2vec2-base-960h) on an unknown dataset.
It achieves the following results on the evaluation set:


- eval_loss: 1.3125
- eval_wer: 0.3139
- eval_runtime: 57.3226
- eval_samples_per_second: 9.996
- eval_steps_per_second: 1.256
- epoch: 18.68
- step: 17000


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


The following hyperparameters were used during training:


- learning_rate: 1e-05
- train_batch_size: 2
- eval_batch_size: 8
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 1000
- num_epochs: 30
- mixed_precision_training: Native AMP


### Framework versions


- Transformers 4.11.3
- Pytorch 1.10.0+cu111
- Datasets 1.13.3
- Tokenizers 0.10.3


model name: wav2vec2-base-myst-demo-colab


model author: samantharhay


task: automatic-speech-recognition


tags: pytorch, tensorboard, wav2vec2, automatic-speech-recognition, transformers, license:apache-2.0, generated_from_trainer, model-index


license: apache-2.0
tags:


- generated_from_trainer
model-index:
name: wav2vec2-base-libir-zenodo


# wav2vec2-base-libir-zenodo


- Loss: 1.4238
- Wer: 0.4336


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


### Training results


| Training Loss | Epoch | Step | Validation Loss | Wer    |
|:-------------:|:-----:|:----:|:---------------:|:------:|
| 3.053         | 1.0   | 31   | 3.1494          | 0.7345 |
| 2.9742        | 2.0   | 62   | 3.0527          | 0.7257 |
| 2.9139        | 3.0   | 93   | 2.8808          | 0.7257 |
| 2.6586        | 4.0   | 124  | 2.6648          | 0.6726 |
| 2.7117        | 5.0   | 155  | 2.4695          | 0.6372 |
| 2.5173        | 6.0   | 186  | 2.3087          | 0.6195 |
| 2.3665        | 7.0   | 217  | 2.2745          | 0.6018 |
| 2.1276        | 8.0   | 248  | 2.2180          | 0.5752 |
| 2.1624        | 9.0   | 279  | 2.1311          | 0.5752 |
| 2.0312        | 10.0  | 310  | 2.0358          | 0.5575 |
| 2.0652        | 11.0  | 341  | 1.9146          | 0.5310 |
| 1.7963        | 12.0  | 372  | 1.8346          | 0.5221 |
| 1.6811        | 13.0  | 403  | 1.8351          | 0.5398 |
| 1.5929        | 14.0  | 434  | 1.8256          | 0.4779 |
| 1.6644        | 15.0  | 465  | 1.7572          | 0.4779 |
| 1.5411        | 16.0  | 496  | 1.8740          | 0.4779 |
| 1.4027        | 17.0  | 527  | 1.5143          | 0.4779 |
| 1.2634        | 18.0  | 558  | 1.3864          | 0.4867 |
| 1.1053        | 19.0  | 589  | 1.3192          | 0.4425 |
| 1.0517        | 20.0  | 620  | 1.4705          | 0.4602 |
| 1.1033        | 21.0  | 651  | 1.6006          | 0.4956 |
| 0.9992        | 22.0  | 682  | 1.4748          | 0.5044 |
| 0.8987        | 23.0  | 713  | 1.3544          | 0.4867 |
| 0.9656        | 24.0  | 744  | 1.2673          | 0.4336 |
| 0.952         | 25.0  | 775  | 1.3955          | 0.4071 |
| 0.8507        | 26.0  | 806  | 1.3520          | 0.4425 |
| 0.8269        | 27.0  | 837  | 1.8992          | 0.4336 |
| 0.7255        | 28.0  | 868  | 1.9850          | 0.4425 |
| 0.8269        | 29.0  | 899  | 3.0089          | 0.4425 |
| 0.6178        | 30.0  | 930  | 1.4238          | 0.4336 |


### Framework versions


model name: wav2vec2-base-libir-zenodo


model author: samantharhay


task: automatic-speech-recognition


- generated_from_trainer
model-index:
- name: wav2vec2-base-timit-demo-colab
results: []


# wav2vec2-base-timit-demo-colab


This model is a fine-tuned version of [facebook/wav2vec2-base-960h](https://huggingface.co/facebook/wav2vec2-base-960h) on the None dataset.
It achieves the following results on the evaluation set:


- Loss: 1.2368
- Wer: 0.8655


- learning_rate: 1e-05
- train_batch_size: 32
- eval_batch_size: 8
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 1000
- num_epochs: 30
- mixed_precision_training: Native AMP


| Training Loss | Epoch | Step | Validation Loss | Wer    |
|:-------------:|:-----:|:----:|:---------------:|:------:|
| 10.7111       | 4.0   | 500  | 6.1429          | 1.0    |
| 4.8905        | 8.0   | 1000 | 6.0597          | 1.0    |
| 3.4516        | 12.0  | 1500 | 3.0125          | 1.0    |
| 2.9895        | 16.0  | 2000 | 2.9629          | 1.0    |
| 2.9155        | 20.0  | 2500 | 2.4479          | 1.0    |
| 2.3186        | 24.0  | 3000 | 1.5888          | 0.9565 |
| 1.8469        | 28.0  | 3500 | 1.2368          | 0.8655 |


model name: wav2vec2-base-timit-demo-colab


license: apache-2.0
tags:


- generated_from_trainer
metrics:
- precision
- recall
- f1
- accuracy
model-index:
- name: bert-finetuned-ner
results: []


# bert-finetuned-ner


This model is a fine-tuned version of [bert-base-cased](https://huggingface.co/bert-base-cased) on an unknown dataset.
It achieves the following results on the evaluation set:


- Loss: 0.1196
- Precision: 0.7872
- Recall: 0.8292
- F1: 0.8077
- Accuracy: 0.9722


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


- learning_rate: 2e-05
- train_batch_size: 8
- eval_batch_size: 8
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- num_epochs: 4


### Training results


| Training Loss | Epoch | Step | Validation Loss | Precision | Recall | F1     | Accuracy |
|:-------------:|:-----:|:----:|:---------------:|:---------:|:------:|:------:|:--------:|
| 0.1243        | 1.0   | 1380 | 0.0932          | 0.6752    | 0.8222 | 0.7415 | 0.9635   |
| 0.0624        | 2.0   | 2760 | 0.0890          | 0.7298    | 0.8368 | 0.7797 | 0.9686   |
| 0.0405        | 3.0   | 4140 | 0.1029          | 0.7792    | 0.8356 | 0.8064 | 0.9715   |
| 0.0226        | 4.0   | 5520 | 0.1196          | 0.7872    | 0.8292 | 0.8077 | 0.9722   |


### Framework versions


- Transformers 4.12.5
- Pytorch 1.10.0+cu111
- Datasets 1.15.1
- Tokenizers 0.10.3


model name: bert-finetuned-ner


model author: tushar-rishav


task: token-classification


tags: pytorch, tensorboard, bert, token-classification, transformers, license:apache-2.0, generated_from_trainer, model-index, autonlp_compatible, infinity_compatible


language: fr
license: mit
datasets:


- Jean-Baptiste/wikiner_fr
widget:
- text: "Boulanger, habitant à Boulanger et travaillant dans le magasin Boulanger situé dans la ville de Boulanger. Boulanger a écrit notamment le très célèbre livre intitulé Boulanger édité par la maison d'édition Boulanger."


# DistilCamemBERT-NER


We present DistilCamemBERT-NER which is [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base) fine tuned for the NER (Named Entity Recognition) task for the French language. The work is inspired by [Jean-Baptiste/camembert-ner](https://huggingface.co/Jean-Baptiste/camembert-ner) based on the [CamemBERT](https://huggingface.co/camembert-base) model. The problem of the modelizations based on CamemBERT is at the scaling moment, for the production phase for example. Indeed, inference cost can be a technological issue. To counteract this effect, we propose this modelization which **divides the inference time by 2** with the same consumption power thanks to [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base).


## Dataset


The dataset used is [wikiner_fr](https://huggingface.co/datasets/Jean-Baptiste/wikiner_fr) which represents ~170k sentences labelized in 5 categories :


* PER: personality ;
* LOC: location ;
* ORG: organization ;
* MISC: miscellaneous entities (movies title, books, etc.) ;
* O: background (Outside entity).


## Evaluation results


| **class**      | **precision (%)** | **recall (%)** | **f1 (%)** | **support (#sub-word)** |
| :------------: | :---------------: | :------------: | :--------: | :---------------------: |
| **global**     | 98.35             | 98.36          | 98.35      | 492'243                 |
| **PER**        | 96.22             | 97.41          | 96.81      | 27'842                  |
| **LOC**        | 93.93             | 93.50          | 93.72      | 31'431                  |
| **ORG**        | 85.13             | 87.08          | 86.10      | 7'662                   |
| **MISC**       | 88.55             | 81.84          | 85.06      | 13'553                  |
| **O**          | 99.40             | 99.55          | 99.47      | 411'755                 |


## Benchmark


This model performance is compared to 2 reference models (see below) with the metric [MCC (Matthews Correlation Coefficient)](https://en.wikipedia.org/wiki/Phi_coefficient). Unfortunately it's impossible to compare with [Jean-Baptiste/camembert-ner](https://huggingface.co/Jean-Baptiste/camembert-ner) model because there is no way to know which observations were used for the evaluation phase. The score is given with a factor x100 and the delta gain with DistilCamemBERT-NER used in reference is in parantheses. For the mean inference time measure, an AMD Ryzen 5 4500U @ 2.3GHz with 6 cores was used:


| **model**                                                                                | **time (ms)**      | **PER**          | **LOC**          | **ORG**          | **MISC**         | **O**            |
| :---------------------------------------------------------------------------------------------------------------: | :----------------: | :--------------: | :--------------: | :--------------: | :--------------: | :------------- : |
| [cmarkea/distilcamembert-base-ner](https://huggingface.co/cmarkea/distilcamembert-base-ner)                       | **43.44**          | **93.91**        | **88.26**        | **84.03**        | **82.74**        | **91.45**        |
| [Davlan/bert-base-multilingual-cased-ner-hrl](https://huggingface.co/Davlan/bert-base-multilingual-cased-ner-hrl) | 87.56
(+102%)  | 79.93
(-15%) | 70.39
(-22%) | 60.26
(-28%) | n/a
(n/a%)   | 69.95
(-24%) |
| [flair/ner-french](https://huggingface.co/flair/ner-french)                                                       | 314.96
(+625%) | 80.18
(-15%) | 72.11
(-18%) | 67.29
(-20%) | 72.39
(-17%) | 74.34
(-19%) |


## How to use DistilCamemBERT-NER


```python
from transformers import pipeline

ner = pipeline(
    task='ner',
    model="cmarkea/distilcamembert-base-ner",
    tokenizer="cmarkea/distilcamembert-base-ner",
    aggregation_strategy="simple"
)
result = ner(
    "Le Crédit Mutuel Arkéa est une banque Française, elle comprend le CMB "
    "qui est une banque située en Bretagne et le CMSO qui est une banque "
    "qui se situe principalement en Aquitaine. C'est sous la présidence de "
    "Louis Lichou, dans les années 1980 que différentes filiales sont créées "
    "au sein du CMB et forment les principales filiales du groupe qui "
    "existent encore aujourd'hui (Federal Finance, Suravenir, Financo, etc.)."
)

result
[{'entity_group': 'ORG',
  'score': 0.99327177,
  'word': 'Crédit Mutuel Arkéa',
  'start': 3,
  'end': 22},
 {'entity_group': 'LOC',
  'score': 0.5869117,
  'word': 'Française',
  'start': 38,
  'end': 47},
 {'entity_group': 'ORG',
  'score': 0.9728106,
  'word': 'CMB',
  'start': 66,
  'end': 69},
 {'entity_group': 'LOC',
  'score': 0.9974824,
  'word': 'Bretagne',
  'start': 99,
  'end': 107},
 {'entity_group': 'ORG',
  'score': 0.956406,
  'word': 'CMSO',
  'start': 114,
  'end': 118},
 {'entity_group': 'LOC',
  'score': 0.99741644,
  'word': 'Aquitaine',
  'start': 169,
  'end': 178},
 {'entity_group': 'PER',
  'score': 0.9988959,
  'word': 'Louis Lichou',
  'start': 208,
  'end': 220},
 {'entity_group': 'ORG',
  'score': 0.93090177,
  'word': 'CMB',
  'start': 291,
  'end': 294},
 {'entity_group': 'ORG',
  'score': 0.9965743,
  'word': 'Federal Finance',
  'start': 374,
  'end': 389},
 {'entity_group': 'ORG',
  'score': 0.99655724,
  'word': 'Suravenir',
  'start': 391,
  'end': 400},
 {'entity_group': 'ORG',
  'score': 0.99653435,
  'word': 'Financo',
  'start': 402,
  'end': 409}]
```


model name: distilcamembert-base-ner


model author: cmarkea


task: token-classification


tags: pytorch, tf, camembert, token-classification, fr, dataset:Jean-Baptiste/wikiner_fr, transformers, license:mit, autonlp_compatible


language: fr
license: mit
datasets:


- oscar
widget:
- text: "J'aime lire les <mask> de SF."


# DistilCamemBERT


We present a distillation version of the well named [CamemBERT](https://huggingface.co/camembert-base), a RoBERTa French model version, alias DistilCamemBERT. The aim of distillation is to drastically reduce the complexity of the model while preserving the performances. The proof of concept is shown in the [DistilBERT paper](https://arxiv.org/abs/1910.01108) and the code used for the training is inspired by the code of [DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/research_projects/distillation).


## Loss function


The training for the distilled model (student model) is designed to be the closest as possible to the original model (teacher model). To perform this the loss function is composed of 3 parts:


* DistilLoss: a distillation loss which measures the silimarity between the probabilities at the outputs of the student and teacher models with a cross-entropy loss on the MLM task ;
* CosineLoss: a cosine embedding loss. This loss function is applied on the last hidden layers of student and teacher models to guarantee a collinearity between them ;
* MLMLoss: and finaly a Masked Language Modeling (MLM) task loss to perform the student model with the original task of the teacher model.


The final loss function is a combination of these three losses functions. We use the following ponderation:


$$Loss = 0.5 \times DistilLoss + 0.3 \times CosineLoss + 0.2 \times MLMLoss$$


## Dataset


To limit the bias between the student and teacher models, the dataset used for the DstilCamemBERT training is the same as the camembert-base training one: OSCAR. The French part of this dataset approximately represents 140 GB on a hard drive disk.


## Training


We pre-trained the model on a nVidia Titan RTX during 18 days.


## Evaluation results


| Dataset name | f1-score |
| :----------: | :------: |
| [FLUE](https://huggingface.co/datasets/flue) CLS     | 83%      |
| [FLUE](https://huggingface.co/datasets/flue) PAWS-X  | 77%      |
| [FLUE](https://huggingface.co/datasets/flue) XNLI    | 77%      |
| [wikiner_fr](https://huggingface.co/datasets/Jean-Baptiste/wikiner_fr) NER | 98%    |


## How to use DistilCamemBERT


Load DistilCamemBERT and its sub-word tokenizer :


```python
from transformers import CamembertModel, CamembertTokenizer

tokenizer = CamembertTokenizer.from_pretrained("cmarkea/distilcamembert-base")
model = CamembertModel.from_pretrained("cmarkea/distilcamembert-base")
model.eval()
...
```


Filling masks using pipeline :


```python
from transformers import pipeline

model_fill_mask = pipeline("fill-mask", model="cmarkea/distilcamembert-base", tokenizer="cmarkea/distilcamembert-base")
results = model_fill_mask("Le camembert est <mask> :)")

results
[{'sequence': '<s> Le camembert est délicieux :)</s>', 'score': 0.3878222405910492, 'token': 7200},
 {'sequence': '<s> Le camembert est excellent :)</s>', 'score': 0.06469205021858215, 'token': 2183},
 {'sequence': '<s> Le camembert est parfait :)</s>', 'score': 0.04534877464175224, 'token': 1654},
 {'sequence': '<s> Le camembert est succulent :)</s>', 'score': 0.04128391295671463, 'token': 26202},
 {'sequence': '<s> Le camembert est magnifique :)</s>', 'score': 0.02425697259604931, 'token': 1509}]
```


model name: distilcamembert-base


model author: cmarkea


task: fill-mask


tags: pytorch, tf, camembert, fill-mask, fr, dataset:oscar, arxiv:1910.01108, transformers, license:mit, autonlp_compatible


language: fr
license: mit
tags:


- zero-shot-classification
- nli
pipeline_tag: zero-shot-classification
widget:
- text: "Selon certains physiciens, un univers parallèle, miroir du nôtre ou relevant de ce que l'on appelle la théorie des branes, autoriserait des neutrons à sortir de notre Univers pour y entrer à nouveau. L'idée a été testée une nouvelle fois avec le réacteur nucléaire de l'Institut Laue-Langevin à Grenoble, plus précisément en utilisant le détecteur de l'expérience Stereo initialement conçu pour chasser des particules de matière noire potentielles, les neutrinos stériles."
candidate_labels: "politique, science, sport, santé"
hypothesis_template: "Ce texte parle de {}."
datasets:
- flue


# DistilCamemBERT-NLI


We present DistilCamemBERT-NLI which is [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base) fine-tuned for the Natural Language Inference (NLI) task for the french language, also known as recognizing textual entailment (RTE). This model is constructed on the XNLI dataset which consists to determine whether a premise entails, contradicts or neither entails nor contradicts a hypothesis.


This modelization is close to [BaptisteDoyen/camembert-base-xnli](https://huggingface.co/BaptisteDoyen/camembert-base-xnli) based on [CamemBERT](https://huggingface.co/camembert-base) model. The problem of the modelizations based on CamemBERT is at the scaling moment, for the production phase for example. Indeed, inference cost can be a technological issue. To counteract this effect, we propose this modelization which divides the inference time by 2 with the same consumption power thanks to DistilCamemBERT.


## Dataset


The dataset XNLI from [FLUE](https://huggingface.co/datasets/flue) is composed of 392,702 premises with their hypothesis for the train and 5,010 couples for the test. The goal is to predict textual entailment (does sentence A imply/contradict/neither sentence B?) and is a classification task (given two sentences, predict one of three labels). The sentence A is called *premise* and sentence B is called *hypothesis*, then the goal of modelization is determined as follows:
$$P(premise=c\in\{contradiction, entailment, neutral\}\vert hypothesis)$$


## Evaluation results


| **class**          | **precision (%)** | **f1-score (%)** | **support** |
| :----------------: | :---------------: | :--------------: | :---------: |
| **global**         | 77.70             | 77.45            | 5,010       |
| **contradiction**  | 78.00             | 79.54            | 1,670       |
| **entailment**     | 82.90             | 78.87            | 1,670       |
| **neutral**        | 72.18             | 74.04            | 1,670       |


## Benchmark


We compare the [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base) model with 2 other modelizations working on french language. The first one [BaptisteDoyen/camembert-base-xnli](https://huggingface.co/BaptisteDoyen/camembert-base-xnli) is based on well named [CamemBERT](https://huggingface.co/camembert-base), the french RoBERTa model and the second one [MoritzLaurer/mDeBERTa-v3-base-mnli-xnli](https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-mnli-xnli) based on [mDeBERTav3](https://huggingface.co/microsoft/mdeberta-v3-base) a multilingual model. To compare the performances the metrics of accuracy and [MCC (Matthews Correlation Coefficient)](https://en.wikipedia.org/wiki/Phi_coefficient) was used and for the mean inference time measure, an **AMD Ryzen 5 4500U @ 2.3GHz with 6 cores** was used:


| **model**          | **time (ms)** | **accuracy (%)** | **MCC (x100)** |
| :--------------: | :-----------: | :--------------: | :------------: |
| [cmarkea/distilcamembert-base-nli](https://huggingface.co/cmarkea/distilcamembert-base-nli) | **51.35**            | 77.45     | 66.24         |
| [BaptisteDoyen/camembert-base-xnli](https://huggingface.co/BaptisteDoyen/camembert-base-xnli) | 105.0              | 81.72     | 72.67         |
| [MoritzLaurer/mDeBERTa-v3-base-mnli-xnli](https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-mnli-xnli) | 299.18 | **83.43** | **75.15**     |


## Zero-shot classification


The main advantage of such modelization is to create a zero-shot classifier allowing text classification without training. This task can be summarized by:
$$P(hypothesis=i\in\mathcal{C}|premise)=\frac{e^{P(premise=entailment\vert hypothesis=i)}}{\sum_{j\in\mathcal{C}}e^{P(premise=entailment\vert hypothesis=j)}}$$


For this part, we use 2 datasets, the first one: [allocine](https://huggingface.co/datasets/allocine) used to train the sentiment analysis models. The dataset is composed of 2 classes: "positif" and "négatif" appreciation of movies reviews. Here we use "Ce commentaire est {}." as the hypothesis template and "positif" and "négatif" as candidate labels.


| **model**     | **time (ms)** | **accuracy (%)** | **MCC (x100)** |
| :--------------: | :-----------: | :--------------: | :------------: |
| [cmarkea/distilcamembert-base-nli](https://huggingface.co/cmarkea/distilcamembert-base-nli) | **195.54**           | 80.59         | 63.71         |
| [BaptisteDoyen/camembert-base-xnli](https://huggingface.co/BaptisteDoyen/camembert-base-xnli) | 378.39             | **86.37**     | **73.74**     |
| [MoritzLaurer/mDeBERTa-v3-base-mnli-xnli](https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-mnli-xnli) | 520.58 | 84.97         | 70.05         |


The second one: [mlsum](https://huggingface.co/datasets/mlsum) used to train the summarization models. We use the articles summary part to predict their topics. In this aim, we aggregate sub-topics and select a few of them. In this case, the hypothesis template used is "C'est un article traitant de {}." and the candidate labels are: "économie", "politique", "sport" and "science".


| **model**        | **time (ms)** |  **accuracy (%)** | **MCC (x100)** |
| :--------------: | :-----------: | :--------------: | :------------: |
| [cmarkea/distilcamembert-base-nli](https://huggingface.co/cmarkea/distilcamembert-base-nli) | **217.77**           | **79.30**     | **70.55**     |
| [BaptisteDoyen/camembert-base-xnli](https://huggingface.co/BaptisteDoyen/camembert-base-xnli) | 448.27             | 70.7          | 64.10         |
| [MoritzLaurer/mDeBERTa-v3-base-mnli-xnli](https://huggingface.co/MoritzLaurer/mDeBERTa-v3-base-mnli-xnli) | 591.34 | 64.45         | 58.67         |


## How to use DistilCamemBERT-NLI


```python
from transformers import pipeline

classifier = pipeline(
    task='zero-shot-classification',
    model="cmarkea/distilcamembert-base-nli",
    tokenizer="cmarkea/distilcamembert-base-nli"
)
result = classifier (
    sequences="Le style très cinéphile de Quentin Tarantino "
    "se reconnaît entre autres par sa narration postmoderne "
    "et non linéaire, ses dialogues travaillés souvent "
    "émaillés de références à la culture populaire, et ses "
    "scènes hautement esthétiques mais d'une violence "
    "extrême, inspirées de films d'exploitation, d'arts "
    "martiaux ou de western spaghetti.",
    candidate_labels="cinéma, technologie, littérature, politique",
    hypothesis_template="Ce texte parle de {}."
)

result
{"labels": ["cinéma",
            "littérature",
            "technologie",
            "politique"],
 "scores": [0.7164115309715271,
            0.12878799438476562,
            0.1092301607131958,
            0.0455702543258667]}
```


model name: distilcamembert-base-nli


model author: cmarkea


task: zero-shot-classification


tags: pytorch, tf, camembert, text-classification, fr, dataset:flue, transformers, license:mit, zero-shot-classification, nli


language: fr
license: mit
datasets:


- amazon_reviews_multi
- allocine
widget:
- text: "Je pensais lire un livre nul, mais finalement je l'ai trouvé super !"
- text: "Cette banque est très bien, mais elle n'offre pas les services de paiements sans contact."
- text: "Cette banque est très bien et elle offre en plus les services de paiements sans contact."


# DistilCamemBERT-Sentiment


We present DistilCamemBERT-Sentiment which is [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base) fine tuned for the sentiment analysis task for the French language. This model is constructed over 2 datasets: [Amazon Reviews](https://huggingface.co/datasets/amazon_reviews_multi) and [Allociné.fr](https://huggingface.co/datasets/allocine) in order to minimize the bias. Indeed, Amazon reviews are very similar in the messages and relatively shorts, contrary to Allociné critics which are long and rich texts.


This modelization is close to [tblard/tf-allocine](https://huggingface.co/tblard/tf-allocine) based on [CamemBERT](https://huggingface.co/camembert-base) model. The problem of the modelizations based on CamemBERT is at the scaling moment, for the production phase for example. Indeed, inference cost can be a technological issue. To counteract this effect, we propose this modelization which **divides the inference time by 2** with the same consumption power thanks to [DistilCamemBERT](https://huggingface.co/cmarkea/distilcamembert-base).


## Dataset


The dataset is composed of 204,993 reviews for training and 4,999 reviews for the test coming from Amazon, and respectively 235,516 and 4,729 critics from [Allocine website](https://www.allocine.fr/). The dataset is labeled into 5 categories:


* 1 star: represents a very bad appreciation,
* 2 stars: bad appreciation,
* 3 stars: neutral appreciation,
* 4 stars: good appreciation,
* 5 stars: very good appreciation.


## Evaluation results


In addition of accuracy (called here *exact accuracy*) in order to be robust to +/-1 star estimation errors, we will take the following definition as a performance measure:


$$\mathrm{top\!-\!2\; acc}=\frac{1}{|\mathcal{O}|}\sum_{i\in\mathcal{O}}\sum_{0\leq l < 2}\mathbb{1}(\hat{f}_{i,l}=y_i)$$


where \\(\hat{f}_l\\) is the l-th largest predicted label, \\(y\\) the true label, \\(\mathcal{O}\\) is the test set of the observations and \\(\mathbb{1}\\) is the indicator function.


| **class**   | **exact accuracy (%)** | **top-2 acc (%)** | **support** |
| :---------: | :--------------------: | :---------------: | :---------: |
| **global**  | 61.01                  | 88.80             | 9,698       |
| **1 star**  | 87.21                  | 77.17             | 1,905       |
| **2 stars** | 79.19                  | 84.75             | 1,935       |
| **3 stars** | 77.85                  | 78.98             | 1,974       |
| **4 stars** | 78.61                  | 90.22             | 1,952       |
| **5 stars** | 85.96                  | 82.92             | 1,932       |


## Benchmark


This model is compared to 3 reference models (see below). As each model doesn't have the same definition of targets, we detail the performance measure used for each of them. For the mean inference time measure, an **AMD Ryzen 5 4500U @ 2.3GHz with 6 cores** was used.


#### bert-base-multilingual-uncased-sentiment


[nlptown/bert-base-multilingual-uncased-sentiment](https://huggingface.co/nlptown/bert-base-multilingual-uncased-sentiment) is based on BERT model in the multilingual and uncased version. This sentiment analyzer is trained on Amazon reviews similarly to our model, hence the targets and their definitions are the same.


| **model** | **time (ms)** | **exact accuracy (%)** | **top-2 acc (%)** |
| :-------: | :-----------: | :--------------------: | :---------------: |
| [cmarkea/distilcamembert-base-sentiment](https://huggingface.co/cmarkea/distilcamembert-base-sentiment) | **95.56** | **61.01** | **88.80** |
| [nlptown/bert-base-multilingual-uncased-sentiment](https://huggingface.co/nlptown/bert-base-multilingual-uncased-sentiment) | 187.70 | 54.41 | 82.82 |


#### tf-allociné and barthez-sentiment-classification


[tblard/tf-allocine](https://huggingface.co/tblard/tf-allocine) based on [CamemBERT](https://huggingface.co/camembert-base) model and [moussaKam/barthez-sentiment-classification](https://huggingface.co/moussaKam/barthez-sentiment-classification) based on [BARThez](https://huggingface.co/moussaKam/barthez) use the same bi-class definition between them. To bring this back to a two-class problem, we will only consider the *"1 star"* and *"2 stars"* labels for the *negative* sentiments and *"4 stars"* and *"5 stars"* for *positive* sentiments. We exclude the *"3 stars"* which can be interpreted as a *neutral* class. In this context, the problem of +/-1 star estimation errors disappears. Then we use only the classical accuracy definition.


| **model** | **time (ms)** | **exact accuracy (%)** |
| :-------: | :-----------: | :--------------------: |
| [cmarkea/distilcamembert-base-sentiment](https://huggingface.co/cmarkea/distilcamembert-base-sentiment) | **95.56** | **97.52** |
| [tblard/tf-allocine](https://huggingface.co/tblard/tf-allocine) | 329.74 | 95.69 |
| [moussaKam/barthez-sentiment-classification](https://huggingface.co/moussaKam/barthez-sentiment-classification) | 197.95 | 94.29 |


## How to use DistilCamemBERT-Sentiment


```python
from transformers import pipeline

analyzer = pipeline(
    task='text-classification',
    model="cmarkea/distilcamembert-base-sentiment",
    tokenizer="cmarkea/distilcamembert-base-sentiment"
)
result = analyzer(
    "J'aime me promener en forêt même si ça me donne mal aux pieds.",
    return_all_scores=True
)

result
[{'label': '1 star',
  'score': 0.047529436647892},
 {'label': '2 stars',
  'score': 0.14150355756282806},
 {'label': '3 stars',
  'score': 0.3586442470550537},
 {'label': '4 stars',
  'score': 0.3181498646736145},
 {'label': '5 stars',
  'score': 0.13417290151119232}]
```


model name: distilcamembert-base-sentiment


model author: cmarkea


task: text-classification


tags: pytorch, tf, camembert, text-classification, fr, dataset:amazon_reviews_multi, dataset:allocine, transformers, license:mit


language:


- en
tags:
- text Classification
license: apache-2.0
widget:
- text: "I like you. </s></s> I love you."


* IMDBSentimentDistilBertModel:


- I have used IMDB movie review dataset to create custom model by using DistilBertForSequenceClassification.


from transformers import DistilBertForSequenceClassification, Trainer, TrainingArguments


model = DistilBertForSequenceClassification.from_pretrained('./imdbsentdistilbertmodel')


model name: imdbsentdistilbertmodel


model author: Sakil


tags: pytorch, distilbert, text-classification, en, transformers, text Classification, license:apache-2.0, infinity_compatible


- en
tags:
- text Classification
license: apache-2.0
widget:
- text: "میں تمہیں پسند کرتا ہوں. </s></s> میں تم سے پیار کرتا ہوں."


* IMDB_URDUSENTIMENT_MODEL
I have used IMDB URDU dataset to create custom model by using DistilBertForSequenceClassification.


model name: IMDB_URDUSENTIMENT_MODEL


test


model name: testmodel


tags:


## license: apache-2.0


model name: test


model author: silentzone


tags: license:apache-2.0


license: apache-2.0
tags:


- generated_from_trainer
metrics:
- bleu
model-index:
- name: t5-small-finetuned-en-to-pt
results: []


<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->


# t5-small-finetuned-en-to-pt


This model is a fine-tuned version of [t5-small](https://huggingface.co/t5-small) on the None dataset.
It achieves the following results on the evaluation set:


- Loss: 0.3295
- Bleu: 5.6807
- Gen Len: 18.6772


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


The following hyperparameters were used during training:


- learning_rate: 0.005
- train_batch_size: 16
- eval_batch_size: 16
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- num_epochs: 10


### Training results


| Training Loss | Epoch | Step  | Validation Loss | Bleu   | Gen Len |
|:-------------:|:-----:|:-----:|:---------------:|:------:|:-------:|
| 0.5787        | 1.0   | 6250  | 0.4928          | 4.1007 | 18.638  |
| 0.5089        | 2.0   | 12500 | 0.4463          | 4.3492 | 18.663  |
| 0.4652        | 3.0   | 18750 | 0.4215          | 4.68   | 18.6652 |
| 0.4353        | 4.0   | 25000 | 0.3980          | 4.8172 | 18.6708 |
| 0.4042        | 5.0   | 31250 | 0.3799          | 4.9719 | 18.6514 |
| 0.3734        | 6.0   | 37500 | 0.3676          | 5.2226 | 18.6572 |
| 0.3396        | 7.0   | 43750 | 0.3513          | 5.2693 | 18.6596 |
| 0.308         | 8.0   | 50000 | 0.3400          | 5.4546 | 18.676  |
| 0.2767        | 9.0   | 56250 | 0.3331          | 5.5649 | 18.6708 |
| 0.2424        | 10.0  | 62500 | 0.3295          | 5.6807 | 18.6772 |


### Framework versions


- Transformers 4.15.0
- Pytorch 1.10.0+cu111
- Datasets 1.18.0
- Tokenizers 0.10.3


model name: t5-small-finetuned-en-to-pt


model author: danhsf


task: text2text-generation


tags: pytorch, tensorboard, t5, text2text-generation, transformers, license:apache-2.0, generated_from_trainer, model-index, autonlp_compatible


license: apache-2.0
tags:


- generated_from_trainer
datasets:
- wmt16
metrics:
- bleu
model-index:
- name: t5-small-finetuned-en-to-ro-lr_2e-3-fp_false
results:
  - task:
name: Sequence-to-sequence Language Modeling
type: text2text-generation
dataset:
name: wmt16
type: wmt16
args: ro-en
metrics:
    - name: Bleu
type: bleu
value: 7.1921


# t5-small-finetuned-en-to-ro-lr_2e-3-fp_false


This model is a fine-tuned version of [t5-small](https://huggingface.co/t5-small) on the wmt16 dataset.
It achieves the following results on the evaluation set:


- Loss: 1.4239
- Bleu: 7.1921
- Gen Len: 18.2611


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


- learning_rate: 0.002
- train_batch_size: 16
- eval_batch_size: 16
- seed: 42
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- num_epochs: 1


### Training results


| Training Loss | Epoch | Step  | Validation Loss | Bleu   | Gen Len |
|:-------------:|:-----:|:-----:|:---------------:|:------:|:-------:|
| 0.8922        | 0.05  | 2000  | 1.7000          | 6.5274 | 18.2656 |
| 0.8621        | 0.1   | 4000  | 1.6409          | 6.6411 | 18.2311 |
| 0.8433        | 0.16  | 6000  | 1.6396          | 6.6601 | 18.2596 |
| 0.8297        | 0.21  | 8000  | 1.6304          | 6.7129 | 18.2581 |
| 0.8006        | 0.26  | 10000 | 1.6022          | 6.6067 | 18.2816 |
| 0.793         | 0.31  | 12000 | 1.5999          | 6.551  | 18.2631 |
| 0.774         | 0.37  | 14000 | 1.5586          | 6.7105 | 18.2661 |
| 0.7618        | 0.42  | 16000 | 1.5769          | 6.7278 | 18.2526 |
| 0.7463        | 0.47  | 18000 | 1.5625          | 6.6972 | 18.2201 |
| 0.7394        | 0.52  | 20000 | 1.5377          | 6.936  | 18.2491 |
| 0.7203        | 0.58  | 22000 | 1.5191          | 7.0205 | 18.2731 |
| 0.7158        | 0.63  | 24000 | 1.5055          | 6.835  | 18.2506 |
| 0.688         | 0.68  | 26000 | 1.4779          | 7.0534 | 18.2716 |
| 0.678         | 0.73  | 28000 | 1.4691          | 6.9735 | 18.2616 |
| 0.6677        | 0.79  | 30000 | 1.4702          | 7.0359 | 18.2496 |
| 0.6568        | 0.84  | 32000 | 1.4534          | 6.9982 | 18.2556 |
| 0.6475        | 0.89  | 34000 | 1.4427          | 7.0443 | 18.2466 |
| 0.6395        | 0.94  | 36000 | 1.4265          | 7.1205 | 18.2721 |
| 0.6319        | 1.0   | 38000 | 1.4239          | 7.1921 | 18.2611 |


### Framework versions


- Transformers 4.12.5
- Pytorch 1.10.0+cu111
- Datasets 1.16.1
- Tokenizers 0.10.3


model name: t5-small-finetuned-en-to-ro-lr_2e-3-fp_false


model author: danhsf


task: text2text-generation


tags: pytorch, tensorboard, t5, text2text-generation, dataset:wmt16, transformers, license:apache-2.0, generated_from_trainer, model-index, autonlp_compatible


tags:


- "es"
- "robust-speech-event"
- generated_from_trainer
datasets:
- common_voice
model-index:
- name: wav2vec2-large-xls-r-300m-spanish-small-v3
results: []


<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->


# wav2vec2-large-xls-r-300m-spanish-small-v3


This model is a fine-tuned version of [jhonparra18/wav2vec2-large-xls-r-300m-spanish-custom](https://huggingface.co/jhonparra18/wav2vec2-large-xls-r-300m-spanish-custom) on the common_voice dataset.
It achieves the following results on the evaluation set:


- Loss: 0.3986
- Wer: 0.1980


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


The following hyperparameters were used during training:


- learning_rate: 0.0004
- train_batch_size: 8
- eval_batch_size: 8
- seed: 42
- gradient_accumulation_steps: 2
- total_train_batch_size: 16
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 500
- num_epochs: 25
- mixed_precision_training: Native AMP


### Training results


| Training Loss | Epoch | Step  | Validation Loss | Wer    |
|:-------------:|:-----:|:-----:|:---------------:|:------:|
| 0.2372        | 0.26  | 400   | 0.3011          | 0.2660 |
| 0.3413        | 0.53  | 800   | 0.3559          | 0.3228 |
| 0.3598        | 0.79  | 1200  | 0.3753          | 0.3400 |
| 0.3529        | 1.05  | 1600  | 0.3385          | 0.2979 |
| 0.3133        | 1.32  | 2000  | 0.3559          | 0.3056 |
| 0.3158        | 1.58  | 2400  | 0.3364          | 0.2994 |
| 0.3092        | 1.85  | 2800  | 0.3210          | 0.2876 |
| 0.2919        | 2.11  | 3200  | 0.3460          | 0.3010 |
| 0.2666        | 2.37  | 3600  | 0.3543          | 0.3036 |
| 0.2819        | 2.64  | 4000  | 0.3477          | 0.2959 |
| 0.283         | 2.9   | 4400  | 0.3492          | 0.2968 |
| 0.2484        | 3.16  | 4800  | 0.3647          | 0.2993 |
| 0.2371        | 3.43  | 5200  | 0.3601          | 0.2942 |
| 0.2382        | 3.69  | 5600  | 0.3656          | 0.3019 |
| 0.2425        | 3.96  | 6000  | 0.3379          | 0.2873 |
| 0.2092        | 4.22  | 6400  | 0.3385          | 0.2736 |
| 0.2171        | 4.48  | 6800  | 0.3503          | 0.2889 |
| 0.2185        | 4.75  | 7200  | 0.3289          | 0.2727 |
| 0.2236        | 5.01  | 7600  | 0.3447          | 0.2771 |
| 0.1882        | 5.27  | 8000  | 0.3586          | 0.2860 |
| 0.1986        | 5.54  | 8400  | 0.3404          | 0.2829 |
| 0.2055        | 5.8   | 8800  | 0.3561          | 0.2869 |
| 0.196         | 6.06  | 9200  | 0.3633          | 0.2811 |
| 0.1748        | 6.33  | 9600  | 0.3703          | 0.2818 |
| 0.1758        | 6.59  | 10000 | 0.3525          | 0.2816 |
| 0.1819        | 6.86  | 10400 | 0.3581          | 0.2765 |
| 0.1715        | 7.12  | 10800 | 0.3480          | 0.2628 |
| 0.1606        | 7.38  | 11200 | 0.3490          | 0.2703 |
| 0.1632        | 7.65  | 11600 | 0.3461          | 0.2706 |
| 0.1638        | 7.91  | 12000 | 0.3458          | 0.2673 |
| 0.1552        | 8.17  | 12400 | 0.3646          | 0.2732 |
| 0.154         | 8.44  | 12800 | 0.3706          | 0.2726 |
| 0.1512        | 8.7   | 13200 | 0.3609          | 0.2683 |
| 0.149         | 8.97  | 13600 | 0.3610          | 0.2668 |
| 0.1357        | 9.23  | 14000 | 0.3693          | 0.2740 |
| 0.1375        | 9.49  | 14400 | 0.3677          | 0.2625 |
| 0.1391        | 9.76  | 14800 | 0.3795          | 0.2762 |
| 0.1378        | 10.02 | 15200 | 0.3541          | 0.2592 |
| 0.1197        | 10.28 | 15600 | 0.3562          | 0.2507 |
| 0.1259        | 10.55 | 16000 | 0.3612          | 0.2584 |
| 0.1266        | 10.81 | 16400 | 0.3470          | 0.2527 |
| 0.1199        | 11.07 | 16800 | 0.3721          | 0.2571 |
| 0.1157        | 11.34 | 17200 | 0.3734          | 0.2571 |
| 0.1107        | 11.6  | 17600 | 0.3730          | 0.2589 |
| 0.1148        | 11.87 | 18000 | 0.3648          | 0.2536 |
| 0.1095        | 12.13 | 18400 | 0.3746          | 0.2521 |
| 0.1047        | 12.39 | 18800 | 0.3566          | 0.2530 |
| 0.1043        | 12.66 | 19200 | 0.3794          | 0.2545 |
| 0.1066        | 12.92 | 19600 | 0.3548          | 0.2439 |
| 0.0974        | 13.18 | 20000 | 0.3702          | 0.2461 |
| 0.0978        | 13.45 | 20400 | 0.3721          | 0.2492 |
| 0.095         | 13.71 | 20800 | 0.3599          | 0.2467 |
| 0.0963        | 13.97 | 21200 | 0.3650          | 0.2402 |
| 0.0902        | 14.24 | 21600 | 0.3689          | 0.2459 |
| 0.0898        | 14.5  | 22000 | 0.3832          | 0.2452 |
| 0.0865        | 14.77 | 22400 | 0.3982          | 0.2436 |
| 0.0911        | 15.03 | 22800 | 0.3785          | 0.2398 |
| 0.0793        | 15.29 | 23200 | 0.3731          | 0.2396 |
| 0.0806        | 15.56 | 23600 | 0.3626          | 0.2372 |
| 0.0789        | 15.82 | 24000 | 0.3707          | 0.2356 |
| 0.0779        | 16.08 | 24400 | 0.3850          | 0.2368 |
| 0.078         | 16.35 | 24800 | 0.3831          | 0.2363 |
| 0.0732        | 16.61 | 25200 | 0.3947          | 0.2287 |
| 0.0733        | 16.88 | 25600 | 0.3928          | 0.2374 |
| 0.0721        | 17.14 | 26000 | 0.3943          | 0.2324 |
| 0.0676        | 17.4  | 26400 | 0.3793          | 0.2311 |
| 0.0682        | 17.67 | 26800 | 0.3958          | 0.2257 |
| 0.0714        | 17.93 | 27200 | 0.3890          | 0.2322 |
| 0.0673        | 18.19 | 27600 | 0.3872          | 0.2229 |
| 0.0613        | 18.46 | 28000 | 0.3828          | 0.2226 |
| 0.0621        | 18.72 | 28400 | 0.3812          | 0.2214 |
| 0.0622        | 18.98 | 28800 | 0.3919          | 0.2212 |
| 0.0576        | 19.25 | 29200 | 0.4000          | 0.2205 |
| 0.0581        | 19.51 | 29600 | 0.3953          | 0.2203 |
| 0.0573        | 19.78 | 30000 | 0.3947          | 0.2190 |
| 0.0576        | 20.04 | 30400 | 0.3909          | 0.2156 |
| 0.0551        | 20.3  | 30800 | 0.4178          | 0.2153 |
| 0.0525        | 20.57 | 31200 | 0.3935          | 0.2152 |
| 0.0522        | 20.83 | 31600 | 0.4054          | 0.2151 |
| 0.0519        | 21.09 | 32000 | 0.3877          | 0.2135 |
| 0.0479        | 21.36 | 32400 | 0.4119          | 0.2107 |
| 0.0472        | 21.62 | 32800 | 0.3967          | 0.2091 |
| 0.048         | 21.89 | 33200 | 0.3812          | 0.2057 |
| 0.0458        | 22.15 | 33600 | 0.3931          | 0.2043 |
| 0.0459        | 22.41 | 34000 | 0.3937          | 0.2049 |
| 0.0448        | 22.68 | 34400 | 0.3900          | 0.2056 |
| 0.0432        | 22.94 | 34800 | 0.4050          | 0.2049 |
| 0.0425        | 23.2  | 35200 | 0.3985          | 0.2014 |
| 0.0415        | 23.47 | 35600 | 0.3976          | 0.2013 |
| 0.0403        | 23.73 | 36000 | 0.4031          | 0.2018 |
| 0.04          | 23.99 | 36400 | 0.3996          | 0.2000 |
| 0.039         | 24.26 | 36800 | 0.3977          | 0.1993 |
| 0.0406        | 24.52 | 37200 | 0.3967          | 0.2000 |
| 0.0391        | 24.79 | 37600 | 0.3986          | 0.1980 |


### Framework versions


- Transformers 4.17.0.dev0
- Pytorch 1.10.2+cu102
- Datasets 1.18.2.dev0
- Tokenizers 0.11.0


model name: wav2vec2-large-xls-r-300m-spanish-small-v3


model author: tomascufaro


task: automatic-speech-recognition


tags: pytorch, wav2vec2, automatic-speech-recognition, dataset:common_voice, transformers, es, robust-speech-event, generated_from_trainer, model-index


language:


- es
license: apache-2.0
tags:
- automatic-speech-recognition
- mozilla-foundation/common_voice_8_0
- generated_from_trainer
- "es"
- "robust-speech-event"
datasets:
- common_voice
model-index:
- name: xls-r-es-test
results: []


# xls-r-es-test


This model is a fine-tuned version of [facebook/wav2vec2-large-xlsr-53](https://huggingface.co/facebook/wav2vec2-large-xlsr-53) on the MOZILLA-FOUNDATION/COMMON_VOICE_8_0 - ES dataset.
It achieves the following results on the evaluation set:


- Loss: 0.1304
- WER: 0.1261
- CER: 0.035


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


- learning_rate: 7.5e-05
- train_batch_size: 8
- eval_batch_size: 8
- seed: 42
- gradient_accumulation_steps: 4
- total_train_batch_size: 32
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 2000
- num_epochs: 10.0
- mixed_precision_training: Native AMP


### Training results


| Training Loss | Epoch | Step  | Validation Loss | Wer    |
|:-------------:|:-----:|:-----:|:---------------:|:------:|
| 2.9613        | 0.07  | 500   | 2.9647          | 1.0    |
| 2.604         | 0.14  | 1000  | 1.8300          | 0.9562 |
| 1.177         | 0.21  | 1500  | 0.3652          | 0.3077 |
| 1.0745        | 0.28  | 2000  | 0.2707          | 0.2504 |
| 1.0103        | 0.35  | 2500  | 0.2338          | 0.2157 |
| 0.9858        | 0.42  | 3000  | 0.2321          | 0.2129 |
| 0.974         | 0.49  | 3500  | 0.2164          | 0.2031 |
| 0.9699        | 0.56  | 4000  | 0.2078          | 0.1970 |
| 0.9513        | 0.63  | 4500  | 0.2173          | 0.2139 |
| 0.9657        | 0.7   | 5000  | 0.2050          | 0.1979 |
| 0.9484        | 0.77  | 5500  | 0.2008          | 0.1919 |
| 0.9317        | 0.84  | 6000  | 0.2012          | 0.1911 |
| 0.9366        | 0.91  | 6500  | 0.2024          | 0.1976 |
| 0.9242        | 0.98  | 7000  | 0.2062          | 0.2028 |
| 0.9138        | 1.05  | 7500  | 0.1924          | 0.1863 |
| 0.921         | 1.12  | 8000  | 0.1935          | 0.1836 |
| 0.9117        | 1.19  | 8500  | 0.1887          | 0.1815 |
| 0.9064        | 1.26  | 9000  | 0.1909          | 0.1839 |
| 0.9118        | 1.32  | 9500  | 0.1869          | 0.1830 |
| 0.9121        | 1.39  | 10000 | 0.1863          | 0.1802 |
| 0.9048        | 1.46  | 10500 | 0.1845          | 0.1791 |
| 0.8955        | 1.53  | 11000 | 0.1863          | 0.1774 |
| 0.8947        | 1.6   | 11500 | 0.1907          | 0.1814 |
| 0.9073        | 1.67  | 12000 | 0.1892          | 0.1853 |
| 0.8927        | 1.74  | 12500 | 0.1821          | 0.1750 |
| 0.8732        | 1.81  | 13000 | 0.1815          | 0.1768 |
| 0.8761        | 1.88  | 13500 | 0.1822          | 0.1749 |
| 0.8751        | 1.95  | 14000 | 0.1789          | 0.1715 |
| 0.8889        | 2.02  | 14500 | 0.1819          | 0.1791 |
| 0.8864        | 2.09  | 15000 | 0.1826          | 0.1794 |
| 0.886         | 2.16  | 15500 | 0.1788          | 0.1776 |
| 0.8915        | 2.23  | 16000 | 0.1756          | 0.1719 |
| 0.8689        | 2.3   | 16500 | 0.1769          | 0.1711 |
| 0.879         | 2.37  | 17000 | 0.1777          | 0.1739 |
| 0.8692        | 2.44  | 17500 | 0.1765          | 0.1705 |
| 0.8504        | 2.51  | 18000 | 0.1699          | 0.1652 |
| 0.8728        | 2.58  | 18500 | 0.1705          | 0.1694 |
| 0.8523        | 2.65  | 19000 | 0.1674          | 0.1645 |
| 0.8513        | 2.72  | 19500 | 0.1661          | 0.1611 |
| 0.8498        | 2.79  | 20000 | 0.1660          | 0.1631 |
| 0.8432        | 2.86  | 20500 | 0.1636          | 0.1610 |
| 0.8492        | 2.93  | 21000 | 0.1708          | 0.1688 |
| 0.8561        | 3.0   | 21500 | 0.1663          | 0.1604 |
| 0.842         | 3.07  | 22000 | 0.1690          | 0.1625 |
| 0.857         | 3.14  | 22500 | 0.1642          | 0.1605 |
| 0.8518        | 3.21  | 23000 | 0.1626          | 0.1585 |
| 0.8506        | 3.28  | 23500 | 0.1651          | 0.1605 |
| 0.8394        | 3.35  | 24000 | 0.1647          | 0.1585 |
| 0.8431        | 3.42  | 24500 | 0.1632          | 0.1573 |
| 0.8566        | 3.49  | 25000 | 0.1614          | 0.1550 |
| 0.8534        | 3.56  | 25500 | 0.1645          | 0.1589 |
| 0.8386        | 3.63  | 26000 | 0.1632          | 0.1582 |
| 0.8357        | 3.7   | 26500 | 0.1631          | 0.1556 |
| 0.8299        | 3.77  | 27000 | 0.1612          | 0.1550 |
| 0.8421        | 3.84  | 27500 | 0.1602          | 0.1552 |
| 0.8375        | 3.91  | 28000 | 0.1592          | 0.1537 |
| 0.8328        | 3.97  | 28500 | 0.1587          | 0.1537 |
| 0.8155        | 4.04  | 29000 | 0.1587          | 0.1520 |
| 0.8335        | 4.11  | 29500 | 0.1624          | 0.1556 |
| 0.8138        | 4.18  | 30000 | 0.1581          | 0.1547 |
| 0.8195        | 4.25  | 30500 | 0.1560          | 0.1507 |
| 0.8092        | 4.32  | 31000 | 0.1561          | 0.1534 |
| 0.8191        | 4.39  | 31500 | 0.1549          | 0.1493 |
| 0.8008        | 4.46  | 32000 | 0.1540          | 0.1493 |
| 0.8138        | 4.53  | 32500 | 0.1544          | 0.1493 |
| 0.8173        | 4.6   | 33000 | 0.1553          | 0.1511 |
| 0.8081        | 4.67  | 33500 | 0.1541          | 0.1484 |
| 0.8192        | 4.74  | 34000 | 0.1560          | 0.1506 |
| 0.8068        | 4.81  | 34500 | 0.1540          | 0.1503 |
| 0.8105        | 4.88  | 35000 | 0.1529          | 0.1483 |
| 0.7976        | 4.95  | 35500 | 0.1507          | 0.1451 |
| 0.8143        | 5.02  | 36000 | 0.1505          | 0.1462 |
| 0.8053        | 5.09  | 36500 | 0.1517          | 0.1476 |
| 0.785         | 5.16  | 37000 | 0.1526          | 0.1478 |
| 0.7936        | 5.23  | 37500 | 0.1489          | 0.1421 |
| 0.807         | 5.3   | 38000 | 0.1483          | 0.1420 |
| 0.8092        | 5.37  | 38500 | 0.1481          | 0.1435 |
| 0.793         | 5.44  | 39000 | 0.1503          | 0.1438 |
| 0.814         | 5.51  | 39500 | 0.1495          | 0.1480 |
| 0.807         | 5.58  | 40000 | 0.1472          | 0.1424 |
| 0.7913        | 5.65  | 40500 | 0.1471          | 0.1422 |
| 0.7844        | 5.72  | 41000 | 0.1473          | 0.1422 |
| 0.7888        | 5.79  | 41500 | 0.1445          | 0.1385 |
| 0.7806        | 5.86  | 42000 | 0.1435          | 0.1394 |
| 0.7773        | 5.93  | 42500 | 0.1461          | 0.1424 |
| 0.786         | 6.0   | 43000 | 0.1450          | 0.1413 |
| 0.7784        | 6.07  | 43500 | 0.1463          | 0.1424 |
| 0.7937        | 6.14  | 44000 | 0.1438          | 0.1386 |
| 0.7738        | 6.21  | 44500 | 0.1437          | 0.1383 |
| 0.7728        | 6.28  | 45000 | 0.1424          | 0.1371 |
| 0.7681        | 6.35  | 45500 | 0.1416          | 0.1376 |
| 0.776         | 6.42  | 46000 | 0.1415          | 0.1380 |
| 0.7773        | 6.49  | 46500 | 0.1416          | 0.1371 |
| 0.7692        | 6.56  | 47000 | 0.1398          | 0.1345 |
| 0.7642        | 6.62  | 47500 | 0.1381          | 0.1341 |
| 0.7692        | 6.69  | 48000 | 0.1392          | 0.1334 |
| 0.7667        | 6.76  | 48500 | 0.1392          | 0.1348 |
| 0.7712        | 6.83  | 49000 | 0.1398          | 0.1333 |
| 0.7628        | 6.9   | 49500 | 0.1392          | 0.1344 |
| 0.7622        | 6.97  | 50000 | 0.1377          | 0.1329 |
| 0.7639        | 7.04  | 50500 | 0.1361          | 0.1316 |
| 0.742         | 7.11  | 51000 | 0.1376          | 0.1327 |
| 0.7526        | 7.18  | 51500 | 0.1387          | 0.1342 |
| 0.7606        | 7.25  | 52000 | 0.1363          | 0.1316 |
| 0.7626        | 7.32  | 52500 | 0.1365          | 0.1313 |
| 0.752         | 7.39  | 53000 | 0.1354          | 0.1309 |
| 0.7562        | 7.46  | 53500 | 0.1362          | 0.1312 |
| 0.7557        | 7.53  | 54000 | 0.1358          | 0.1325 |
| 0.7588        | 7.6   | 54500 | 0.1343          | 0.1311 |
| 0.7485        | 7.67  | 55000 | 0.1346          | 0.1301 |
| 0.7466        | 7.74  | 55500 | 0.1354          | 0.1314 |
| 0.7558        | 7.81  | 56000 | 0.1359          | 0.1325 |
| 0.7578        | 7.88  | 56500 | 0.1363          | 0.1334 |
| 0.7411        | 7.95  | 57000 | 0.1346          | 0.1301 |
| 0.7478        | 8.02  | 57500 | 0.1355          | 0.1305 |
| 0.7451        | 8.09  | 58000 | 0.1349          | 0.1302 |
| 0.7383        | 8.16  | 58500 | 0.1349          | 0.1294 |
| 0.7482        | 8.23  | 59000 | 0.1341          | 0.1293 |
| 0.742         | 8.3   | 59500 | 0.1338          | 0.1296 |
| 0.7343        | 8.37  | 60000 | 0.1348          | 0.1307 |
| 0.7385        | 8.44  | 60500 | 0.1324          | 0.1282 |
| 0.7567        | 8.51  | 61000 | 0.1334          | 0.1281 |
| 0.7342        | 8.58  | 61500 | 0.1338          | 0.1289 |
| 0.7401        | 8.65  | 62000 | 0.1331          | 0.1285 |
| 0.7362        | 8.72  | 62500 | 0.1329          | 0.1283 |
| 0.7241        | 8.79  | 63000 | 0.1323          | 0.1277 |
| 0.7244        | 8.86  | 63500 | 0.1317          | 0.1269 |
| 0.7274        | 8.93  | 64000 | 0.1308          | 0.1260 |
| 0.7411        | 9.0   | 64500 | 0.1309          | 0.1256 |
| 0.7255        | 9.07  | 65000 | 0.1316          | 0.1265 |
| 0.7406        | 9.14  | 65500 | 0.1315          | 0.1270 |
| 0.7418        | 9.21  | 66000 | 0.1315          | 0.1269 |
| 0.7301        | 9.27  | 66500 | 0.1315          | 0.1273 |
| 0.7248        | 9.34  | 67000 | 0.1323          | 0.1274 |
| 0.7423        | 9.41  | 67500 | 0.1309          | 0.1267 |
| 0.7152        | 9.48  | 68000 | 0.1312          | 0.1271 |
| 0.7295        | 9.55  | 68500 | 0.1306          | 0.1262 |
| 0.7231        | 9.62  | 69000 | 0.1308          | 0.1263 |
| 0.7344        | 9.69  | 69500 | 0.1313          | 0.1267 |
| 0.7264        | 9.76  | 70000 | 0.1305          | 0.1263 |
| 0.7309        | 9.83  | 70500 | 0.1303          | 0.1262 |
| 0.73          | 9.9   | 71000 | 0.1303          | 0.1261 |
| 0.7353        | 9.97  | 71500 | 0.1304          | 0.1260 |


### Framework versions


- Transformers 4.17.0.dev0
- Pytorch 1.10.2+cu102
- Datasets 1.18.3
- Tokenizers 0.11.0


model name: xls-r-es-test


model author: tomascufaro


task: automatic-speech-recognition


tags: pytorch, wav2vec2, automatic-speech-recognition, es, dataset:common_voice, transformers, license:apache-2.0, mozilla-foundation/common_voice_8_0, generated_from_trainer, robust-speech-event, model-index


license: apache-2.0
tags:


- generated_from_trainer
datasets:
- common_voice
model-index:
- name: wav2vec2-large-xls-r-300m-spanish-custom
results: []


<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->


# wav2vec2-large-xls-r-300m-spanish-custom


This model is a fine-tuned version of [facebook/wav2vec2-xls-r-300m](https://huggingface.co/facebook/wav2vec2-xls-r-300m) on the common_voice dataset.
It achieves the following results on the evaluation set:


- Loss: 0.4426
- Wer: 0.2117


## Model description


More information needed


## Intended uses & limitations


## Training and evaluation data


## Training procedure


### Training hyperparameters


The following hyperparameters were used during training:


- learning_rate: 0.0003
- train_batch_size: 8
- eval_batch_size: 8
- seed: 42
- gradient_accumulation_steps: 2
- total_train_batch_size: 16
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 500
- num_epochs: 30
- mixed_precision_training: Native AMP


### Training results