update for local pipeline

parent_app/app.py

@@ -1,118 +0,0 @@
-import time
-
-from hub import (
-    setup_dataset_on_hub,
-    duplicate_space_on_hub,
-    add_project_config_to_space_repo,
-)
-
-import streamlit as st
-
-
-# Constants
-# Written here to avoid defaults.py
-DEFAULT_DOMAIN = "farming"
-
-st.set_page_config(
-    "Domain Data Grower", page_icon="🧑‍🌾", initial_sidebar_state="collapsed"
-)
-
-st.header("🧑‍🌾 Domain Data Grower")
-st.divider()
-
-st.sidebar.link_button(
-    "🤗 Get your Hub Token", "https://huggingface.co/settings/tokens"
-)
-
-################################################################################
-# APP MARKDOWN
-################################################################################
-
-st.header("🌱 Create a domain specific dataset")
-
-st.markdown(
-    """This space will set up your domain specific dataset project. It will 
-create the resources that you need to build a dataset. Those resources include: 
-    
-- A dataset repository on the Hub
-- Another space to define expert domain and run generation pipelines    
-
-For a complete overview of the project. Check out the README 
-"""
-)
-
-st.page_link(
-    "pages/🧑‍🌾 Domain Data Grower.py",
-    label="Domain Data Grower",
-    icon="🧑‍🌾",
-)
-
-################################################################################
-# CONFIGURATION
-################################################################################
-
-st.subheader("🌾 Project Configuration")
-
-project_name = st.text_input("Project Name", DEFAULT_DOMAIN)
-hub_username = st.text_input("Hub Username", "argilla")
-hub_token = st.text_input("Hub Token", type="password")
-private_selector = st.checkbox("Private Space", value=False)
-
-if st.button("🤗 Setup Project Resources"):
-    repo_id = f"{hub_username}/{project_name}"
-
-    setup_dataset_on_hub(
-        repo_id=repo_id,
-        hub_token=hub_token,
-    )
-
-    st.success(
-        f"Dataset seed created and pushed to the Hub. Check it out [here](https://huggingface.co/datasets/{hub_username}/{project_name}).  Hold on the repo_id: {repo_id}, we will need it in the next steps."
-    )
-
-    space_name = f"{project_name}_config_space"
-
-    duplicate_space_on_hub(
-        source_repo="argilla/domain-specific-datasets-template",
-        target_repo=space_name,
-        hub_token=hub_token,
-        private=private_selector,
-    )
-
-    st.success(
-        f"Configuration Space created. Check it out [here](https://huggingface.co/spaces/{hub_username}/{space_name})."
-    )
-
-    argilla_name = f"{project_name}_argilla_space"
-
-    duplicate_space_on_hub(
-        source_repo="argilla/argilla-template-space",
-        target_repo=argilla_name,
-        hub_token=hub_token,
-        private=private_selector,
-    )
-
-    st.success(
-        f"Argilla Space created. Check it out [here](https://huggingface.co/spaces/{hub_username}/{argilla_name})."
-    )
-
-    seconds = 5
-
-    with st.spinner(f"Adding project configuration to spaces in {seconds} seconds"):
-        time.sleep(seconds)
-        add_project_config_to_space_repo(
-            dataset_repo_id=repo_id,
-            hub_token=hub_token,
-            project_name=project_name,
-            argilla_space_repo_id=f"{hub_username}/{argilla_name}",
-            project_space_repo_id=f"{hub_username}/{space_name}",
-        )
-
-    st.subheader("👢 Next Steps")
-
-    st.write("Go to you project specific space!")
-
-    st.link_button(
-        "🧑‍🌾 Open Configuration Space",
-        f"https://huggingface.co/spaces/{hub_username}/{space_name}",
-    )

parent_app/hub.py

@@ -1,76 +0,0 @@
-import json
-from tempfile import mktemp
-
-
-from huggingface_hub import duplicate_space, HfApi
-
-
-hf_api = HfApi()
-
-
-def setup_dataset_on_hub(repo_id, hub_token):
-    # create an empty dataset repo on the hub
-    hf_api.create_repo(
-        repo_id=repo_id,
-        token=hub_token,
-        repo_type="dataset",
-    )
-
-    # upload the seed data
-    hf_api.upload_file(
-        path_or_fileobj="seed_data.json",
-        path_in_repo="seed_data.json",
-        repo_id=repo_id,
-        repo_type="dataset",
-        token=hub_token,
-    )
-
-
-def duplicate_space_on_hub(source_repo, target_repo, hub_token, private=False):
-    duplicate_space(
-        from_id=source_repo,
-        to_id=target_repo,
-        token=hub_token,
-        private=private,
-        exist_ok=True,
-    )
-
-
-def add_project_config_to_space_repo(
-    dataset_repo_id,
-    hub_token,
-    project_name,
-    argilla_space_repo_id,
-    project_space_repo_id,
-):
-    #  upload the seed data and readme to the hub
-
-    with open("project_config.json", "w") as f:
-        json.dump(
-            {
-                "project_name": project_name,
-                "argilla_space_repo_id": argilla_space_repo_id,
-                "project_space_repo_id": project_space_repo_id,
-                "dataset_repo_id": dataset_repo_id,
-            },
-            f,
-        )
-
-    hf_api.upload_file(
-        path_or_fileobj="project_config.json",
-        path_in_repo="project_config.json",
-        token=hub_token,
-        repo_id=project_space_repo_id,
-        repo_type="space",
-    )
-
-
-def pull_seed_data_from_repo(repo_id, hub_token):
-    tempfile_path = mktemp()
-    # pull the dataset repo from the hub
-    hf_api.hf_hub_download(
-        repo_id=repo_id, token=hub_token, repo_type="dataset", filename=tempfile_path
-    )
-    return json.load(open(tempfile_path))
-
-

parent_app/pages/🧑‍🌾 Domain Data Grower.py

@@ -1,15 +0,0 @@
-import streamlit as st
-import requests
-
-
-readme_location = "https://raw.githubusercontent.com/huggingface/data-is-better-together/51f29e67165d8277d9f9d1e4be60869f4b705a08/domain-specific-datasets/README.md"
-
-
-def open_markdown_file(url):
-    response = requests.get(url)
-    return response.text
-
-
-readme = open_markdown_file(readme_location)
-
-st.markdown(readme)

parent_app/project_config.json

@@ -1 +0,0 @@
-{"project_name": "farming", "argilla_space_repo_id": "ignacioct/farming_argilla_space", "project_space_repo_id": "ignacioct/farming_config_space", "dataset_repo_id": "ignacioct/farming"}

parent_app/seed_data.json

@@ -1,16 +0,0 @@
-{
-  "domain": "farming",
-  "perspectives": [
-    "Family Farming"
-  ],
-  "topics": [
-    "animal welfare"
-  ],
-  "examples": [
-    {
-      "question": "Compare and contrast the environmental footprint of industrial and small-scale farming.",
-      "answer": "Regenerative agriculture practices aim to restore soil health through methods that increase soil organic matter, enhance microbial activity, and improve soil structure. These practices include no-till farming, cover cropping, diverse crop rotations, and integrated livestock management. According to LaCanne and Lundgren (2018), soil health improves due to increased biodiversity and organic matter, enhancing its water retention and nutrient efficiency. Moreover, Jones (2012) in \"Soil carbon & organic farming\" reports that these practices significantly elevate biodiversity, both above and below the soil surface, promoting resilient ecosystems and agroecological balances."
-    }
-  ],
-  "domain_expert_prompt": "You will be asked about family farming and agribusiness related topics, from different perspectives.\n Your answer should be logical and supported by facts, don't fabricate arguments. \n Try to gather a diverse point of view taking into account current theories in agronomy, biology, economics, anthropology and ecology."
-}