:books: Add documentation items

.idea/gpt-fr-cased-small.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,55 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="28">
+            <item index="0" class="java.lang.String" itemvalue="batcher" />
+            <item index="1" class="java.lang.String" itemvalue="tensorboard" />
+            <item index="2" class="java.lang.String" itemvalue="tqdm" />
+            <item index="3" class="java.lang.String" itemvalue="spacy" />
+            <item index="4" class="java.lang.String" itemvalue="spacy-conll" />
+            <item index="5" class="java.lang.String" itemvalue="pydevd-pycharm" />
+            <item index="6" class="java.lang.String" itemvalue="transformers" />
+            <item index="7" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="8" class="java.lang.String" itemvalue="torch" />
+            <item index="9" class="java.lang.String" itemvalue="torchvision" />
+            <item index="10" class="java.lang.String" itemvalue="m2r" />
+            <item index="11" class="java.lang.String" itemvalue="networkx" />
+            <item index="12" class="java.lang.String" itemvalue="mlflow" />
+            <item index="13" class="java.lang.String" itemvalue="nltk" />
+            <item index="14" class="java.lang.String" itemvalue="pytorch-lightning" />
+            <item index="15" class="java.lang.String" itemvalue="PyYAML" />
+            <item index="16" class="java.lang.String" itemvalue="torchtext" />
+            <item index="17" class="java.lang.String" itemvalue="tensorflow" />
+            <item index="18" class="java.lang.String" itemvalue="sentencepiece" />
+            <item index="19" class="java.lang.String" itemvalue="tensorflow-gpu" />
+            <item index="20" class="java.lang.String" itemvalue="pandas" />
+            <item index="21" class="java.lang.String" itemvalue="datasets" />
+            <item index="22" class="java.lang.String" itemvalue="tensorflow_hub" />
+            <item index="23" class="java.lang.String" itemvalue="cython-0.29.21" />
+            <item index="24" class="java.lang.String" itemvalue="sentence-transformers" />
+            <item index="25" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="26" class="java.lang.String" itemvalue="gensim" />
+            <item index="27" class="java.lang.String" itemvalue="scikit_learn" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="W605" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnreachableCodeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="processCode" value="true" />
+      <option name="processLiterals" value="true" />
+      <option name="processComments" value="true" />
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/gpt-fr-cased-small.iml" filepath="$PROJECT_DIR$/.idea/gpt-fr-cased-small.iml" />
+    </modules>
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+</project>

README.md

@@ -17,7 +17,7 @@ license: apache-2.0
 
 <img src="imgs/logo.png" width="200">
 
-**GPT-fr** is a French GPT trained on a very large and heterogeneous French corpus. Models of different sizes are trained using the new CNRS  (French National Centre for Scientific Research) [Jean Zay](http://www.idris.fr/eng/jean-zay/) supercomputer. We released the pre-trained weights for the following model sizes:
+**GPT-fr** is a French GPT trained on a very large and heterogeneous French corpus. We trained models of different sizes and released the pre-trained weights for the following model sizes:
 
 | Model name | Number of layers | Attention Heads | Embedding Dimension | Total Parameters |
 | :------:       |   :---: | :---: | :---: | :---: |
@@ -26,12 +26,13 @@ license: apache-2.0
 
 ## Intended uses & limitations
 
-GPT is a generative model which can be leveraged for language generation tasks. Besides, many tasks may be formatted such that the output is directly generated in natural language. Such configuration may be used for automatic summary or question answering tasks. 
+GPT is a generative model which can be leveraged for language generation tasks. Besides, many tasks may be formatted such that the output is directly generated in natural language. Such configuration may be used for tasks such as automatic summary or question answering tasks. We do hope our model might be use for both academic and industrial applications. 
 
 #### How to use
 
+The model might be used through the astonishing `Transformers librairie:
+
 ```python
-import torch
 from transformers import GPT2Tokenizer, GPT2LMHeadModel
 
 # Load pretrained model and tokenizer
@@ -53,14 +54,13 @@ beam_outputs = model.generate(
     num_return_sequences=1
 )
 
-print("Output:\\\\\\\\\\\\\\\\
-" + 100 * '-')
+print("Output:\n" + 100 * '-')
 print(tokenizer.decode(beam_outputs[0], skip_special_tokens=True))
 ```
 
 #### Limitations and bias
 
-Large pre-trained language models tend to reproduce the biases from the dataset used for pre-training, in particular gender discrimination. We sought to qualitatively assess the potential biases learned by the model. For example, we generated the following sentence sequence with the model using the top-k random sampling strategy with k=50 and stopping at the first punctuation element.  "Ma femme/Mon mari vient d'obtenir un nouveau poste en tant qu'\\_\\_\\_\\_\\_\\_\\_":
+Large pre-trained language models tend to reproduce the biases from the dataset used for pre-training, in particular gender discrimination. As we do hope our model might be use for both academic and industrial applications, we try to limit such shortcoming effects by paying a specific attention to our pre-training corpus and filter explicit and offensive content. The corpus building steps are detailed in our paper. Nonetheless, we sought to qualitatively assess the potential biases learned by the model. We do appreciate your feedback to better assess such effects. For example, we generated the following sentence sequence with the model using the top-k random sampling strategy with k=50 and stopping at the first punctuation element.  "Ma femme/Mon mari vient d'obtenir un nouveau poste en tant qu'\_\_\_\_\_\_\_":
 
 The position generated for the wife are:
 
@@ -84,11 +84,11 @@ The position generated for the husband are:
 
 ## Training data
 
-We created a dedicated corpus to train our generative model. Indeed the model uses a fixed-length context size of 1,024 and require long documents to be trained.  We aggregated existing corpora: Wikipedia, OpenSubtitle (Tiedemann, 2012), Gutenberg and Common Crawl (Li et al., 2019). Corpora are filtered and separated into sentences. Successive sentences are then concatenated within the limit of 1024 tokens per document.
+We created a dedicated corpus to train our generative model. Indeed the model uses a fixed-length context size of 1,024 and require long documents to be trained.  We aggregated existing corpora: [Wikipedia](https://dumps.wikimedia.org/frwiki/), [OpenSubtitle](http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2016/mono/) ([Tiedemann, 2012](#tiedemann-2012)), [Gutenberg](http://www.gutenberg.org). Corpora are filtered and separated into sentences. Successive sentences are then concatenated within the limit of 1,024 tokens per document.
 
 ## Training procedure
 
-We pre-trained the model on a TPU v2-8 using the Google Colab inter-server.
+We pre-trained the model on a TPU v2-8 using the amazing [Google Colab](https://colab.research.google.com) inter-server.
 
 ## Eval results
 
@@ -102,4 +102,7 @@ In line with the [WikiText](https://blog.einstein.ai/the-wikitext-long-term-depe
 @inproceedings{...,
   year={2020}
 }
-```
+```
+### References
+
+><div name="tiedemann-2012">Jörg Tiedemann: Parallel Data, Tools and Interfaces in OPUS. LREC 2012: 2214-2218</div>