use full competition code

Dockerfile

@@ -1,2 +1,82 @@
-FROM huggingface/competitions:latest
+FROM nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    TZ=UTC \
+    HF_HUB_ENABLE_HF_TRANSFER=1
+
+ENV PATH="${HOME}/miniconda3/bin:${PATH}"
+ARG PATH="${HOME}/miniconda3/bin:${PATH}"
+
+RUN mkdir -p /tmp/model && \
+    chown -R 1000:1000 /tmp/model && \
+    mkdir -p /tmp/data && \
+    chown -R 1000:1000 /tmp/data
+
+RUN apt-get update &&  \
+    apt-get upgrade -y &&  \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    curl \
+    ca-certificates \
+    gcc \
+    git \
+    locales \
+    net-tools \
+    wget \
+    libpq-dev \
+    libsndfile1-dev \
+    git \
+    git-lfs \
+    libgl1 \
+    unzip \
+    openjdk-11-jre-headless \
+    libseccomp-dev \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
+
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+    git lfs install
+
+WORKDIR /app
+
+RUN mkdir -p /app/.cache
+ENV HF_HOME="/app/.cache"
+RUN chown -R 1000:1000 /app
+USER 1000
+ENV HOME=/app
+
+ENV PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    GRADIO_ALLOW_FLAGGING=never \
+    GRADIO_NUM_PORTS=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_THEME=huggingface \
+    SYSTEM=spaces
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && sh Miniconda3-latest-Linux-x86_64.sh -b -p /app/miniconda \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh
+ENV PATH /app/miniconda/bin:$PATH
+
+RUN conda create -p /app/env -y python=3.10
+
+SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
+
+RUN conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia && \
+    conda clean -ya && \
+    conda install -c "nvidia/label/cuda-12.1.0" cuda-nvcc && conda clean -ya
+
+COPY --chown=1000:1000 . /app/
+RUN make sandbox
+
+# give permissions to run sandbox
+RUN chmod +x /app/sandbox
+
+ENV PATH="/app:${PATH}"
+
+RUN pip install -U pip
+RUN pip install -e .
+RUN pip install -r requirements_docker.txt
+
 CMD uvicorn competitions.app:app --host 0.0.0.0 --port 7860 --workers 1

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2022] [Hugging Face Inc.]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,33 @@
+CFLAGS += -std=c99 -Wall -O2
+LDFLAGS += -lseccomp
+.PHONY: quality style test
+
+quality:
+	python -m black --check --line-length 119 --target-version py38 .
+	python -m isort --check-only .
+	python -m flake8 --max-line-length 119
+
+style:
+	python -m black --line-length 119 --target-version py38 .
+	python -m isort .
+
+docker:
+	docker build -t competitions:latest .
+	docker tag competitions:latest huggingface/competitions:latest
+	docker push huggingface/competitions:latest
+
+test:
+	pytest -sv .
+
+sandbox: sandbox.c
+	gcc $(CFLAGS) $^ -o $@ $(LDFLAGS)
+
+clean:
+	rm *.so sandbox
+
+pip:
+	rm -rf build/
+	rm -rf dist/
+	make style && make quality
+	python setup.py sdist bdist_wheel
+	twine upload dist/* --verbose

README_competitions.md

@@ -0,0 +1,29 @@
+# Competitions
+
+Create a machine learning competition for your organization, friends or the world!
+
+## Getting Started
+
+There are two types of competitions you can create:
+
+- generic: a competition where you provide the data and the participants provide the predictions as a CSV file. all the test data is always available to the participants.
+
+- script: a competition where you provide the data and the participants provide the code that generates the predictions. test data can be hidden from the participants.
+
+## Costs
+
+Creating a competition is free. However, you will need to pay for the compute resources used to run the competition. The cost of the compute resources depends the type of competition you create.
+
+- generic: generic competitions are free to run. you can, however, upgrade the compute to cpu-basic to speed up the metric calculation and reduce the waiting time for the participants.
+
+- script: script competitions require a compute resource to run the participant's code. you can choose between a variety of cpu and gpu instances (T4, A10g and even A100). the cost of the compute resource is charged per hour.
+
+For information on the cost of the compute resources, please see the [pricing page](https://huggingface.co/docs/hub/spaces-overview#hardware-resources).
+
+## Visibility
+
+You can choose to make your competition public or private. Public competitions are visible to everyone and anyone can participate. Private competitions are only visible to the people you invite!
+
+## How to create a competition?
+
+Please read the [docs](https://huggingface.co/docs/competitions) to learn how to create a competition.

competitions

@@ -0,0 +1 @@
+Subproject commit 447fff164c586ae3d42617c7726d8563514c3f17

docs/source/_toctree.yml

@@ -0,0 +1,24 @@
+- sections: 
+  - local: index
+    title: 🤗 Competitions
+  - local: pricing
+    title: Pricing
+  title: Get started
+- sections:
+  - local: create_competition
+    title: Create competition
+  - local: competition_repo
+    title: Competition repo
+  - local: competition_space
+    title: Competition space
+  - local: custom_metric
+    title: Custom metric
+  title: Organizing
+- sections:
+  - local: submit
+    title: Making a submission
+  - local: leaderboard
+    title: Leaderboard
+  - local: teams
+    title: Teams
+  title: Participating

docs/source/competition_repo.mdx

@@ -0,0 +1,157 @@
+# Competition Repo
+
+NOTE: Competition repo must always be kept private. Do NOT make it public!
+
+The competition repo consists of the following files:
+
+```
+├── COMPETITION_DESC.md
+├── conf.json
+├── DATASET_DESC.md
+├── solution.csv
+├── SUBMISSION_DESC.md
+├── submission_info
+│   └── *.json
+├── submissions
+│   └── *.csv
+├── teams.json
+└── user_team.json
+```
+
+### COMPETITION_DESC.md
+
+This file contains the description of the competition. It is a markdown file.
+You can use the markdown syntax to format the text and modify the file according to your needs.
+Competition description is shown on the front page of the competition.
+
+### DATASET_DESC.md
+
+This file contains the description of the dataset. It is again a markdown file.
+This file is used to describe the dataset and is shown on the dataset page.
+In this file you can mention which columns are present in the dataset, what is the meaning of each column, what is the format of the dataset, etc.
+
+### conf.json
+
+conf.json is the configuration file for the competition. An example conf.json is shown below:
+
+```
+{
+   "COMPETITION_TYPE":"generic",
+   "SUBMISSION_LIMIT":5,
+   "TIME_LIMIT": 10,
+   "HARDWARE":"cpu-basic",
+   "SELECTION_LIMIT":10,
+   "END_DATE":"2024-05-25",
+   "EVAL_HIGHER_IS_BETTER":1,
+   "SUBMISSION_ID_COLUMN":"id",
+   "SUBMISSION_COLUMNS":"id,pred",
+   "SUBMISSION_ROWS":10000,
+   "EVAL_METRIC":"roc_auc_score",
+   "LOGO":"https://github.com/abhishekkrthakur/public_images/blob/main/song.png?raw=true",
+   "DATASET": "",
+   "SUBMISSION_FILENAMES": ["submission.csv"],
+   "SCORING_METRIC": "roc_auc_score"
+}
+```
+
+This file is created when you create a new competition. You can modify this file according to your needs.
+However, we do not recommend changing the evaluation metric field once the competition has started
+as it would require you to re-evaluate all the submissions.
+
+- COMPETITION_TYPE: This field is used to specify the type of competition. Currently, we support two types of competitions: `generic` and `script`.
+    - `generic` competition is a competition where the users can submit a csv file (or a different format) and the submissions are evaluated using a metric.
+    - `script` competition is a competition where the users can submit a huggingface model repo containing a script.py. The script.py is run to generate submission.csv which is then evaluated using a metric.
+- SUBMISSION_LIMIT: This field is used to specify the number of submissions a user can make in a day.
+- TIME_LIMIT: This field is used to specify the time limit for each submission in seconds. (used only for `script` competitions)
+- HARDWARE: This field is used to specify the hardware on which the submissions will be evaluated.
+- SELECTION_LIMIT: This field is used to specify the number of submissions that will be selected for the leaderboard. (used only for `script` competitions)
+- END_DATE: This field is used to specify the end date of the competition. The competition will be automatically closed on the end date. Private leaderboard will be made available on the end date.
+- EVAL_HIGHER_IS_BETTER: This field is used to specify if the evaluation metric is higher the better or lower the better. If the value is 1, then higher the better. If the value is 0, then lower the better.
+- SUBMISSION_ID_COLUMN: This field is used to specify the name of the id column in the submission file.
+- SUBMISSION_COLUMNS: This field is used to specify the names of the columns in the submission file. The names must be comma separated without any spaces.
+- SUBMISSION_ROWS: This field is used to specify the number of rows in the submission file without the header.
+- EVAL_METRIC: This field is used to specify the evaluation metric. We support all the scikit-learn metrics and even custom metrics.
+- LOGO: This field is used to specify the logo of the competition. The logo must be a png file. The logo is shown on the all pages of the competition.
+- DATASET: This field is used to specify the PRIVATE dataset used in the competition. The dataset is available to the users only during the script run. This is only used for script competitions.
+- SUBMISSION_FILENAMES: This field is used to specify the name of the submission file. This is only used for script competitions with custom metrics and must not be changed for generic competitions.
+- SCORING_METRIC: When using a custom metric / multiple metrics, this field is used to specify the metric name that will be used for scoring the submissions.
+
+### solution.csv
+
+This file contains the solution for the competition. It is a csv file. A sample is shown below:
+
+```
+id,pred,split
+0,1,public
+1,0,private
+2,0,private
+3,1,private
+4,0,public
+5,1,private
+6,1,public
+7,1,private
+8,0,public
+9,0,private
+10,0,private
+11,0,private
+12,1,private
+13,0,private
+14,1,public
+````
+
+The solution file is used to evaluate the submissions. The solution file must always have an id column and a split column.
+The split column is used to split the solution into public and private parts. The split column can have two values: `public` and `private`.
+You can have multiple columns in the solution file. However, the evaluation metric must support multiple columns.
+
+For example, if the evaluation metric is `roc_auc_score` then the solution file must have two columns: `id` and `pred`.
+The names of id and pred columns can be anything. The names will be grabbed from the `conf.json` file.
+Please make sure you have appropriate column names in the `conf.json` file and that you have both public and private splits in the solution file.
+
+### SUBMISSION_DESC.md
+
+This file contains the description of the submission. It is a markdown file.
+You can use the markdown syntax to format the text and modify the file according to your needs.
+Submission description is shown on the submission page.
+
+Here you can mention the format of the submission file, what columns are required in the submission file, etc.
+
+For the example solution file shown above, the submission file must have two columns: `id` and `pred`.
+An example of sample_submission.csv is shown below:
+
+```
+id,pred
+0,0.6
+1,0.1
+2,0.5
+3,1.6
+4,0.8
+5,1
+6,1
+7,1
+8,0
+9,0
+10,0.1
+11,0.4
+12,1.9
+13,0.01
+14,1.1
+```
+
+When a user submits a submission file, the system will check if the submission file has the required columns.
+If the submission file does not have the required columns, the submission will be rejected.
+
+It is the responsibility of the organzier to make sure they provide a sample submission file in correct format and a submission description file.
+
+### submission_info
+
+This folder contains the submission info files. Each submission info file contains the information about a submission.
+This folder is created when a first submission is made. The submission info files are json files.
+
+### submissions
+
+This folder contains the submissions made by the users. Each submission is a csv file (or a different format).
+This folder is created when a first submission is made.
+
+### other files
+
+The other files teams.json and user_team.json are used to store the information about the teams.

docs/source/competition_space.mdx

@@ -0,0 +1,38 @@
+# Competition Space
+
+A competition space is a Hugging Face Space where the actual competition takes place. It is a space where you can submit your model and get a score. It is also a space where competitors can see the leaderboard, discuss, and make submissions.
+
+Check out an example competition space below:
+
+![competition space](https://github.com/abhishekkrthakur/public_images/blob/main/competition_space.png?raw=true)
+
+The competition space consists of the following:
+
+- Competition description (content is fetched from private competition repo)
+- Dataset description (content is fetched from private competition repo)
+- Leaderboard (content is fetched from private competition repo)
+    - Public (available to everyone, all the time)
+    - Private (available to everyone, but only after the competition ends)
+- Submissions
+    - Submission guidelines (content is fetched from private competition repo)
+    - My submissions (users can see their own submissions)
+    - New submission (users can make new submissions)
+- Discussions (accessible via community tab)
+
+### Secrets
+
+The competition space requires two secrets:
+
+- `HF_TOKEN`: this is the Hugging Face write token of the user who created the competition space. This token must be kept alive for the duration of the competition. In case the token expires, the competition space will stop working. If you change/refresh/delete this token, you will need to update this secret.
+- `COMPETITION_ID`: this is the path of private competition repo. e.g. `abhishek/private-competition-data`. If you change the name of the private competition repo, you will need to update this secret.
+
+Note: The above two secrets are crucial for the competition space to work!
+
+### Public & private competition spaces
+
+A competition space can be public or private. A public competition space is available to everyone, all the time. 
+A private competition space is only available to the members of the organization the competition space is created in.
+
+You can at any point make the competition public.
+
+Generally, we recommend testing every aspect of the competition space in a private competition space before making it public.

docs/source/create_competition.mdx

@@ -0,0 +1,35 @@
+# Create a competition
+
+Creating a competition is super easy and you have full control over the data, evaluation metric and the hardware used.
+
+To create a competition, you need to have a [Hugging Face](https://huggingface.co) account. You will also need a write token which will be used throughout the competition to upload data and submissions.
+Please note that the write token is private and should not be shared with anyone and must not be refreshed during the course of the competition.
+In case you decide to refresh the token, you will need to update the token in the competition space's settings otherwise the competitors will not be able to upload submissions and competition will stop working.
+You can find/generate a write token [here](https://huggingface.co/settings/tokens).
+
+To create a competition, you also need an organization. You can either create a new organization or use an existing one that you are already a member of.
+You can create a new organization [here](https://huggingface.co/organizations/new).
+
+To create a competition, click [here](https://huggingface.co/spaces/competitions/create).
+
+### Competition creator
+
+![competition](https://github.com/abhishekkrthakur/public_images/blob/main/competition_creator.png?raw=true)
+
+Please note: you will be able to change almost every setting later on. However, we dont recommend changing evaluation metric once the competition has started.
+As it will require you to re-evaluate all the submissions.
+
+
+### Types of competitions
+
+- generic: generic competitions are competitions where the participants submit a CSV file (or any other file) containing the predictions for the whole test set.
+The predictions are then evaluated against solution.csv (or a solution file) using the evaluation metric provided by the competition creator.
+These competitions are easy to setup and free to host (if you use cpu-basic). You can improve the evaluation runtime by upgrading generic competitions to cpu-upgrade.
+For generic competition, all the test data (without labels) is available to the participants all the time.
+
+- script: script competitions are competitions where the participants submit a python script that takes in the test set and outputs the predictions.
+The predictions are then evaluated against solution.csv (or a solution file) using the evaluation metric provided by the competition creator.
+These competitions are only free to host if you use cpu-basic as the backend for evaluation, and this is not recommended!
+In script competition, the test data can be kept private. The participants wont be able to see the test data at all.
+The participants submit a huggingface model repo containing `script.py` which is run to generate predictions on hidden test data.
+

docs/source/custom_metric.mdx

@@ -0,0 +1,149 @@
+# Custom metric
+
+In case you don't settle for the default scikit-learn metrics, you can define your own metric.
+
+Here, we expect the organizer to know python. 
+
+### How to define a custom metric
+
+To define a custom metric, change `EVAL_METRIC` in `conf.json` to `custom`. You must also make sure that `EVAL_HIGHER_IS_BETTER` is set to `1` or `0` depending on whether a higher value of the metric is better or not.
+
+The second step is to create a file `metric.py` in the private competition repo.
+The file should contain a `compute` function that takes competition params as input.
+
+Here is the part where we check if metric is custom and calculate the metric value:
+
+```python
+def compute_metrics(params):
+    if params.metric == "custom":
+        metric_file = hf_hub_download(
+            repo_id=params.competition_id,
+            filename="metric.py",
+            token=params.token,
+            repo_type="dataset",
+        )
+        sys.path.append(os.path.dirname(metric_file))
+        metric = importlib.import_module("metric")
+        evaluation = metric.compute(params)
+    .
+    .
+    .
+````
+
+You can find the above part in competitions github repo `compute_metrics.py`
+
+`params` is defined as:
+
+```python
+class EvalParams(BaseModel):
+    competition_id: str
+    competition_type: str
+    metric: str
+    token: str
+    team_id: str
+    submission_id: str
+    submission_id_col: str
+    submission_cols: List[str]
+    submission_rows: int
+    output_path: str
+    submission_repo: str
+    time_limit: int
+    dataset: str  # private test dataset, used only for script competitions
+```
+
+You are free to do whatever you want to in the `compute` function. 
+In the end it must return a dictionary with the following keys:
+
+```python
+{
+    "public_score": {
+        "metric1": metric_value,
+    },,
+    "private_score": {
+        "metric1": metric_value,
+    },,
+}
+```
+
+public and private scores must be dictionaries! You can also use multiple metrics.
+Example for multiple metrics:
+
+```python
+{
+    "public_score": {
+        "metric1": metric_value,
+        "metric2": metric_value,
+    },
+    "private_score": {
+        "metric1": metric_value,
+        "metric2": metric_value,
+    },
+}
+```
+
+Note: When using multiple metrics, conf.json must have `SCORING_METRIC` specified to rank the participants in the competition.
+
+For example, if I want to use metric2 to rank the participants, I will set `SCORING_METRIC` to `metric2` in `conf.json`.
+
+### Example of a custom metric
+
+```python
+import pandas as pd
+from huggingface_hub import hf_hub_download
+
+
+def compute(params):
+    solution_file = hf_hub_download(
+        repo_id=params.competition_id,
+        filename="solution.csv",
+        token=params.token,
+        repo_type="dataset",
+    )
+
+    solution_df = pd.read_csv(solution_file)
+
+    submission_filename = f"submissions/{params.team_id}-{params.submission_id}.csv"
+    submission_file = hf_hub_download(
+        repo_id=params.competition_id,
+        filename=submission_filename,
+        token=params.token,
+        repo_type="dataset",
+    )
+    submission_df = pd.read_csv(submission_file)
+
+    public_ids = solution_df[solution_df.split == "public"][params.submission_id_col].values
+    private_ids = solution_df[solution_df.split == "private"][params.submission_id_col].values
+
+    public_solution_df = solution_df[solution_df[params.submission_id_col].isin(public_ids)]
+    public_submission_df = submission_df[submission_df[params.submission_id_col].isin(public_ids)]
+
+    private_solution_df = solution_df[solution_df[params.submission_id_col].isin(private_ids)]
+    private_submission_df = submission_df[submission_df[params.submission_id_col].isin(private_ids)]
+
+    public_solution_df = public_solution_df.sort_values(params.submission_id_col).reset_index(drop=True)
+    public_submission_df = public_submission_df.sort_values(params.submission_id_col).reset_index(drop=True)
+
+    private_solution_df = private_solution_df.sort_values(params.submission_id_col).reset_index(drop=True)
+    private_submission_df = private_submission_df.sort_values(params.submission_id_col).reset_index(drop=True)
+
+    # CALCULATE METRICS HERE.......
+    # _metric = SOME METRIC FUNCTION
+    target_cols = [col for col in solution_df.columns if col not in [params.submission_id_col, "split"]]
+    public_score = _metric(public_solution_df[target_cols], public_submission_df[target_cols])
+    private_score = _metric(private_solution_df[target_cols], private_submission_df[target_cols])
+
+    evaluation = {
+        "public_score": {
+            "metric1": public_score,
+        },
+        "private_score": {
+            "metric1": public_score,
+        }
+    }
+    return evaluation
+```
+
+Take a careful look at the above code. 
+You can see that we are downloading the solution file and the submission file from the dataset repo.
+We are then calculating the metric on the public and private splits of the solution and submission files. 
+Finally, we are returning the metric values in a dictionary.

docs/source/index.mdx

@@ -0,0 +1,34 @@
+# Competitions
+
+Create a machine learning competition for your organization, friends or the world!
+
+![competition](https://github.com/abhishekkrthakur/public_images/blob/main/competition_page.png?raw=true)
+
+## Supported competition types
+
+There are two types of competitions you can create:
+
+- generic: a competition where you provide the data and the participants provide the predictions as a CSV file. all the test data is always available to the participants.
+
+- script: a competition where you provide the data and the participants provide the code that generates the predictions. test data can be hidden from the participants.
+
+You can choose to make your competition public or private. Public competitions are visible to everyone and anyone can participate. Private competitions are only visible to the people you invite!
+
+## Why choose Hugging Face Competitions?
+
+- you can create totally private competitions that are only visible to the people you invite.
+
+- generic competition can be hosted for free.
+
+- script competitions have a variety of compute options to choose from: CPU, T4, A10g & even A100.
+
+- you have full control over the data you want to use for your competition.
+
+- its open source!
+
+
+## Issues / feature requests
+
+Something missing? Found a bug? Please open an issue on this [GitHub repository](https://github.com/huggingface/competitions) and we'll fix it as soon as possible!
+
+To host competition on hf.co/competitions, please contact us at autotrain [at] huggingface [dot] co

docs/source/leaderboard.mdx

@@ -0,0 +1,14 @@
+# Understanding the leaderboard
+
+There are two types of leaderboards for all competitions:
+
+- Public Leaderboard: This leaderboard is calculated on X% of the test dataset, and is what you see on the competition page all the time.
+The value of X will be mentioned in the problem statement by the organizers.
+
+- Private Leaderboard: This leaderboard is calculated on the remaining (100-X)% of the test dataset, and is made public only after the competition is over.
+Your final ranking is based on the private leaderboard.
+
+When you make a submission, both the public and private leaderboard scores are calculated but only the public leaderboard score is displayed.
+When a competition is over, the private leaderboard score is revealed and the final rankings are calculated based on that.
+You can select N submissions to be considered for the private leaderboard score calculation.
+The value of N will be mentioned in the problem statement by the organizers.

docs/source/pricing.mdx

@@ -0,0 +1,12 @@
+# Pricing
+
+Creating a competition is free. However, you will need to pay for the compute resources used to run the competition. The cost of the compute resources depends the type of competition you create.
+
+- generic: generic competitions are free to run. you can, however, upgrade the compute to cpu-upgrade to speed up the metric calculation and reduce the waiting time for the participants.
+
+- script: script competitions require a compute resource to run the participant's code. you can choose between a variety of cpu and gpu instances (T4, A10g and even A100). the cost of the compute resource is charged per minute.
+
+For script competitions, the script submitted by the participants will run without internet for a time limit provided by the organizer.
+You will be charged for the compute resources used by the participants. The cost of the compute resources depends on the type of compute resource you choose.
+
+For information on the cost of the compute resources, please see the [pricing page](https://huggingface.co/docs/hub/spaces-overview#hardware-resources).

docs/source/submit.mdx

@@ -0,0 +1,24 @@
+# Making a submission
+
+The submission format and example submissions are usually provided by the competition organizer.
+This page describes how to make submissions using the competition UI.
+
+To make a submission, you must provide a read-only token that allows the competition to access user info.
+
+### `generic` competition: submitting a file
+
+- provide your hugginface read-only token
+- upload a file
+- add a description
+- click "Submit"
+
+![submit_generic](https://github.com/abhishekkrthakur/public_images/blob/main/submission_generic.png?raw=true)
+
+### `script` competition: submitting a model repo with `script.py`
+
+- provide your hugginface read-only token
+- provide a link to your model repo (e.g. abhishek/submission-model-1). The model repo must contain a `script.py` file that generates submission in the required format.
+- add a description
+- click "Submit"
+
+![submit_script](https://github.com/abhishekkrthakur/public_images/blob/main/submission_script.png?raw=true)

docs/source/teams.mdx

@@ -0,0 +1,3 @@
+# Teaming up
+
+Coming soon!

examples/generate_fake_submissions.py

@@ -0,0 +1,67 @@
+import io
+import json
+import os
+import random
+from datetime import datetime
+
+from huggingface_hub import HfApi
+from tqdm import tqdm
+
+
+NUM_USERS = 1000
+NUM_SUBMISSIONS = 10
+MIN_SCORE = 0.0
+MAX_SCORE = 1.0
+START_DATE = datetime(2022, 1, 1, 0, 0, 0)
+END_DATE = datetime(2022, 11, 23, 0, 0, 0)
+COMPETITION_ID = os.environ.get("COMPETITION_ID")
+AUTOTRAIN_TOKEN = os.environ.get("AUTOTRAIN_TOKEN")
+
+if __name__ == "__main__":
+    # example submission:
+    # {"name": "abhishek", "id": "5fa19f4ba13e063b8b2b5e11", "submissions": [{"date": "2022-11-09", "time": "12:54:55", "submission_id": "c0eed646-838f-482f-bf4d-2c651f8de43b", "submission_comment": "", "status": "pending", "selected": true, "public_score": -1, "private_score": -1}, {"date": "2022-11-09", "time": "14:02:21", "submission_id": "bc6b08c2-c684-4ee1-9be2-35cf717ce618", "submission_comment": "", "status": "done", "selected": true, "public_score": 0.3333333333333333, "private_score": 0.3333333333333333}, {"date": "2022-11-17", "time": "21:31:04", "submission_id": "4a2984b1-de10-411d-be0f-9aa7be16245f", "submission_comment": "", "status": "done", "selected": false, "public_score": 0.3333333333333333, "private_score": 0.3333333333333333}]}
+
+    for i in tqdm(range(NUM_USERS)):
+        name = f"test_{i}"
+        # generate random id
+        id = "".join(random.choices("0123456789abcdef", k=24))
+        submissions = []
+        for j in range(NUM_SUBMISSIONS):
+            date = START_DATE + (END_DATE - START_DATE) * random.random()
+            time = date.strftime("%H:%M:%S")
+            date = date.strftime("%Y-%m-%d")
+            submission_id = "".join(random.choices("0123456789abcdef", k=36))
+            submission_comment = ""
+            status = "done"
+            selected = False
+            public_score = MIN_SCORE + (MAX_SCORE - MIN_SCORE) * random.random()
+            private_score = MIN_SCORE + (MAX_SCORE - MIN_SCORE) * random.random()
+            submission = {
+                "date": date,
+                "time": time,
+                "submission_id": submission_id,
+                "submission_comment": submission_comment,
+                "status": status,
+                "selected": selected,
+                "public_score": public_score,
+                "private_score": private_score,
+            }
+            submissions.append(submission)
+
+        submission = {
+            "name": name,
+            "id": id,
+            "submissions": submissions,
+        }
+        fname = f"{id}.json"
+        user_submission_info_json = json.dumps(submission)
+        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=user_submission_info_json_buffer,
+            path_in_repo=fname,
+            repo_id=COMPETITION_ID,
+            repo_type="dataset",
+            token=AUTOTRAIN_TOKEN,
+        )

requirements.txt

@@ -0,0 +1,17 @@
+# essentials
+fastapi==0.111.0
+joblib==1.4.2
+loguru==0.7.2
+pandas==2.2.2
+huggingface_hub==0.24.6
+tabulate==0.9.0
+markdown==3.6
+psutil==6.0.0
+python-multipart==0.0.9
+uvicorn==0.30.1
+py7zr==0.21.1
+pydantic==2.8.2
+gradio==4.37.2
+authlib==1.3.1
+itsdangerous==2.2.0
+hf-transfer

requirements_docker.txt

@@ -0,0 +1,17 @@
+# machine learning libs
+scikit-learn==1.5.1
+jiwer==3.0.4
+sacremoses==0.1.1
+sentencepiece==0.2.0
+xgboost==2.1.0
+nltk==3.8.1
+optuna==3.6.1
+einops==0.8.0
+peft==0.12.0
+trl==0.9.6
+tiktoken==0.7.0
+transformers==4.44.1
+accelerate==0.33.0
+diffusers==0.30.0
+bitsandbytes==0.43.3
+tqdm==4.66.4

sandbox.c

@@ -0,0 +1,59 @@
+#include <errno.h>
+#include <seccomp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+int main(int argc, char* argv[]) {
+    if (argc < 2) {
+        fprintf(stderr, "Usage: %s <command> [args...]\n", argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    scmp_filter_ctx ctx;
+
+    // Initialize the seccomp filter in blocklist mode
+    ctx = seccomp_init(SCMP_ACT_ALLOW);
+    if (ctx == NULL) {
+        perror("seccomp_init");
+        return EXIT_FAILURE;
+    }
+
+    // Block relevant network-related syscalls, so as to block egress internet access
+
+    // We cannot deny these calls as they are needed by cuda
+    // This should not be a big deal for our use case if what we want is to block egress network access
+    // (just blocking connect should actually be enough)
+
+    // seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(socket), 0);
+    // seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(bind), 0);
+    // seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(listen), 0);
+
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(connect), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(accept), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(send), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(sendto), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(sendmsg), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(recv), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(recvfrom), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(recvmsg), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(setsockopt), 0);
+    seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(getsockopt), 0);
+
+    // Load the filter into the kernel
+    if (seccomp_load(ctx) < 0) {
+        perror("seccomp_load");
+        seccomp_release(ctx);
+        return EXIT_FAILURE;
+    }
+
+#ifdef DEBUG
+    printf("seccomp filter installed. Network access is blocked.\n");
+#endif
+
+    // Execute the target program
+    execvp(argv[1], argv + 1);
+
+    seccomp_release(ctx);
+    return EXIT_SUCCESS;
+}

setup.cfg

@@ -0,0 +1,16 @@
+[metadata]
+license_file = LICENSE
+version = attr: competitions.__version__
+
+[isort]
+ensure_newline_before_comments = True
+force_grid_wrap = 0
+include_trailing_comma = True
+line_length = 119
+lines_after_imports = 2
+multi_line_output = 3
+use_parentheses = True
+
+[flake8]
+ignore = E203, E501, W503
+max-line-length = 119

setup.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+# Copyright 2022 Hugging Face Inc
+#
+# Lint as: python3
+# pylint: enable=line-too-long
+"""Hugging Face Competitions
+"""
+import os
+
+from setuptools import find_packages, setup
+
+
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(this_directory, "README.md"), encoding="utf-8") as f:
+    LONG_DESCRIPTION = f.read()
+
+QUALITY_REQUIRE = [
+    "black~=23.0",
+    "isort==5.13.2",
+    "flake8==7.0.0",
+    "mypy==1.8.0",
+]
+
+TEST_REQUIRE = ["pytest", "pytest-cov"]
+
+EXTRAS_REQUIRE = {
+    "dev": QUALITY_REQUIRE,
+    "quality": QUALITY_REQUIRE,
+    "test": QUALITY_REQUIRE + TEST_REQUIRE,
+    "docs": QUALITY_REQUIRE + TEST_REQUIRE + ["hf-doc-builder"],
+}
+
+with open("requirements.txt", encoding="utf-8") as f:
+    INSTALL_REQUIRES = f.read().splitlines()
+
+setup(
+    name="competitions",
+    description="Hugging Face Competitions",
+    long_description=LONG_DESCRIPTION,
+    author="HuggingFace Inc.",
+    url="https://github.com/huggingface/competitions",
+    download_url="https://github.com/huggingface/competitions/tags",
+    packages=find_packages("."),
+    entry_points={"console_scripts": ["competitions=competitions.cli.competitions:main"]},
+    install_requires=INSTALL_REQUIRES,
+    extras_require=EXTRAS_REQUIRE,
+    python_requires=">=3.10",
+    license="Apache 2.0",
+    classifiers=[
+        "Intended Audience :: Developers",
+        "Intended Audience :: Education",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python :: 3.10",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    ],
+    keywords="huggingface competitions machine learning ai nlp tabular",
+    data_files=[
+        (
+            "templates",
+            [
+                "competitions/templates/index.html",
+            ],
+        ),
+    ],
+    include_package_data=True,
+)