Merge branch 'main' into feat/choose-models

README.md

@@ -80,7 +80,10 @@ pip install synthetic-dataset-generator
 
 ### Environment Variables
 
-- `HF_TOKEN`: Your Hugging Face token to push your datasets to the Hugging Face Hub and run Inference Endpoints Requests. You can get one [here](https://huggingface.co/settings/tokens/new?ownUserPermissions=repo.content.read&ownUserPermissions=repo.write&globalPermissions=inference.serverless.write&tokenType=fineGrained).
+- `HF_TOKEN`: Your Hugging Face token to push your datasets to the Hugging Face Hub and run *Free* Inference Endpoints Requests. You can get one [here](https://huggingface.co/settings/tokens/new?ownUserPermissions=repo.content.read&ownUserPermissions=repo.write&globalPermissions=inference.serverless.write&tokenType=fineGrained).
+
+Optionally, you can also push your datasets to Argilla for further curation by setting the following environment variables:
+
 - `ARGILLA_API_KEY`: Your Argilla API key to push your datasets to Argilla.
 - `ARGILLA_API_URL`: Your Argilla API URL to push your datasets to Argilla.
 
@@ -90,6 +93,12 @@ pip install synthetic-dataset-generator
 python app.py
 ```
 
+### Argilla integration
+
+Argilla is a open source tool for data curation. It allows you to annotate and review datasets, and push curated datasets to the Hugging Face Hub. You can easily get started with Argilla by following the [quickstart guide](https://docs.argilla.io/latest/getting_started/quickstart/).
+
+![Argilla integration](https://huggingface.co/spaces/argilla/synthetic-data-generator/resolve/main/assets/argilla.png)
+
 ## Custom synthetic data generation?
 
 Each pipeline is based on distilabel, so you can easily change the LLM or the pipeline steps.

src/distilabel_dataset_generator/apps/base.py

@@ -475,6 +475,27 @@ def get_success_message_row() -> gr.Markdown:
 
 def show_success_message(org_name, repo_name) -> gr.Markdown:
     client = get_argilla_client()
+    if client is None:
+        return gr.Markdown(
+            value="""
+            <div style="padding: 1em; background-color: #e6f3e6; border-radius: 5px; margin-top: 1em;">
+                <h3 style="color: #2e7d32; margin: 0;">Dataset Published Successfully!</h3>
+                <p style="margin-top: 0.5em;">
+                The generated dataset is in the right format for fine-tuning with TRL, AutoTrain, or other frameworks. Your dataset is now available at:
+                <a href="https://huggingface.co/datasets/{org_name}/{repo_name}" target="_blank" style="color: #1565c0; text-decoration: none;">
+                    https://huggingface.co/datasets/{org_name}/{repo_name}
+                    </a>
+                </p>
+                <p style="margin-top: 1em; font-size: 0.9em; color: #333;">
+                    By configuring an `ARGILLA_API_URL` and `ARGILLA_API_KEY` you can curate the dataset in Argilla.
+                    Unfamiliar with Argilla? Here are some docs to help you get started:
+                    <br>• <a href="https://docs.argilla.io/latest/getting_started/quickstart/" target="_blank">How to get started with Argilla</a>
+                    <br>• <a href="https://docs.argilla.io/latest/how_to_guides/annotate/" target="_blank">How to curate data in Argilla</a>
+                    <br>• <a href="https://docs.argilla.io/latest/how_to_guides/import_export/" target="_blank">How to export data once you have reviewed the dataset</a>
+                </p>
+            </div>
+            """
+        )
     argilla_api_url = client.api_url
     return gr.Markdown(
         value=f"""

src/distilabel_dataset_generator/apps/eval.py

@@ -334,8 +334,10 @@ def push_dataset(
     push_dataset_to_hub(dataframe, org_name, repo_name, oauth_token, private)
     try:
         progress(0.1, desc="Setting up user and workspace")
-        client = get_argilla_client()
         hf_user = HfApi().whoami(token=oauth_token.token)["name"]
+        client = get_argilla_client()
+        if client is None:
+            return ""
         if eval_type == "ultrafeedback":
             num_generations = len((dataframe["generations"][0]))
             fields = [

src/distilabel_dataset_generator/apps/sft.py

@@ -220,8 +220,10 @@ def push_dataset(
     push_dataset_to_hub(dataframe, org_name, repo_name, oauth_token, private)
     try:
         progress(0.1, desc="Setting up user and workspace")
-        client = get_argilla_client()
         hf_user = HfApi().whoami(token=oauth_token.token)["name"]
+        client = get_argilla_client()
+        if client is None:
+            return ""
         if "messages" in dataframe.columns:
             settings = rg.Settings(
                 fields=[

src/distilabel_dataset_generator/apps/textcat.py

@@ -58,7 +58,10 @@ def generate_system_prompt(dataset_description, temperature, progress=gr.Progres
     labels = data["labels"]
     return system_prompt, labels
 
-def generate_sample_dataset(system_prompt, difficulty, clarity, labels, num_labels, progress=gr.Progress()):
+
+def generate_sample_dataset(
+    system_prompt, difficulty, clarity, labels, num_labels, progress=gr.Progress()
+):
     dataframe = generate_dataset(
         system_prompt=system_prompt,
         difficulty=difficulty,
@@ -138,11 +141,7 @@ def generate_dataset(
     # create final dataset
     distiset_results = []
     for result in labeller_results:
-        record = {
-            key: result[key]
-            for key in ["labels", "text"]
-            if key in result
-        }
+        record = {key: result[key] for key in ["labels", "text"] if key in result}
         distiset_results.append(record)
 
     dataframe = pd.DataFrame(distiset_results)
@@ -212,13 +211,16 @@ def push_dataset(
     push_dataset_to_hub(
         dataframe, org_name, repo_name, num_labels, labels, oauth_token, private
     )
+
     dataframe = dataframe[
         (dataframe["text"].str.strip() != "") & (dataframe["text"].notna())
     ]
     try:
         progress(0.1, desc="Setting up user and workspace")
-        client = get_argilla_client()
         hf_user = HfApi().whoami(token=oauth_token.token)["name"]
+        client = get_argilla_client()
+        if client is None:
+            return ""
         labels = get_preprocess_labels(labels)
         settings = rg.Settings(
             fields=[