add app

.gitignore

@@ -0,0 +1,229 @@
+run-backend.ps
+.eslintrc.js
+.venv
+.*credentials.json
+.credentials.json
+/*.ipynb
+logbook.md
+
+.transl_sysprompt_en-nl
+
+# ignore compiled styles
+*.css
+
+# dependencies
+**/node_modules/
+**/.pnp
+*.pnp.js
+
+# testing
+/coverage
+
+# VSCode
+**/.vscode/
+
+# production
+**/build/
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+
+# python
+data/
+Pipfile*
+
+# .idea (JetBrains)
+**/.idea/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+test.py
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties``
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser

README.md

@@ -1,7 +1,7 @@
 ---
 title: Steps Calculator
-emoji: 🐨
-colorFrom: red
+emoji: 🦶
+colorFrom: orange
 colorTo: yellow
 sdk: gradio
 sdk_version: 4.22.0

app.py

@@ -0,0 +1,110 @@
+from math import ceil
+
+import gradio as gr
+from datasets import load_dataset, IterableDataset
+from transformers import AutoTokenizer, PreTrainedTokenizer
+
+
+def count_tokens(batch, tokenizer, text_column):
+    encoded = tokenizer(batch[text_column])
+    return {"num_tokens": [len(input_ids) for input_ids in encoded["input_ids"]]}
+
+
+def get_dataset_num_tokens(
+    dataset: IterableDataset, tokenizer: PreTrainedTokenizer, text_column: str, progress=gr.Progress()
+) -> int:
+    progress((0, None), desc="Counting tokens", unit="tokens")
+    ds = dataset.map(
+        count_tokens, batched=True, batch_size=1000, fn_kwargs={"tokenizer": tokenizer, "text_column": text_column}
+    )
+
+    total_num_tokens = 0
+    for sample in ds:
+        total_num_tokens += sample["num_tokens"]
+        progress((total_num_tokens, None), desc="Counting tokens", unit="tokens")
+
+    return total_num_tokens
+
+
+def calculate_steps(
+    dataset_name: str,
+    dataset_split: str,
+    dataset_config: str | None,
+    tokenizer_name: str,
+    num_gpus_per_node: int,
+    num_nodes: int,
+    batch_size: int,
+    grad_accum: int,
+    block_size: int,
+    text_column: str = "text",
+    token: str | None = None,
+):
+    dataset_config = None if not dataset_config.strip() else dataset_config
+    text_column = "text" if not text_column.strip() else text_column
+    token = None if not token.strip() else token
+    try:
+        dataset = load_dataset(dataset_name, dataset_config, streaming=True, token=token, split=dataset_split)
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, token=token)
+        total_num_tokens = get_dataset_num_tokens(dataset, tokenizer, text_column)
+    except Exception as exc:
+        raise gr.Error(str(exc))
+    else:
+        dataset_size = ceil(total_num_tokens / block_size)
+        world_size = num_gpus_per_node * num_nodes
+        num_steps = ceil(dataset_size / (world_size * batch_size * grad_accum))
+        return dataset_size, num_steps
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        """# Steps Calculator
+    
+Calculate the number of steps required to run through your whole dataset with a given sequence length. This is \
+especially useful when training with a streaming dataset and you're not sure how many steps you need to run through \
+the dataset with a given tokenizer and block size."""
+    )
+
+    with gr.Row():
+        dataset_name = gr.Text(label="Dataset name")
+        dataset_split = gr.Text(label="Dataset split", value="train")
+        dataset_config = gr.Text(label="Dataset config (optional)")
+        tokenizer_name = gr.Text(label="Tokenizer name")
+
+    with gr.Row():
+        num_gpus_per_node = gr.Number(value=1, minimum=1, label="Number of GPUs per node")
+        num_nodes = gr.Number(value=1, minimum=1, label="Number of nodes")
+        batch_size = gr.Number(value=8, minimum=1, label="Batch size")
+        grad_accum = gr.Number(value=1, minimum=1, label="Gradient accumulation steps")
+        block_size = gr.Number(value=2048, minimum=1, label="Block size")
+        text_column = gr.Text(value="text", label="Text column")
+        token = gr.Text(label="HF acces token (optional)")
+
+    with gr.Row():
+        with gr.Column():
+            calculate_btn = gr.Button(value="Calculate")
+        with gr.Column():
+            samples = gr.Number(value=None, minimum=1, label="Total block-sized samples", interactive=False)
+            steps = gr.Number(value=None, minimum=1, label="Total steps needed", interactive=False)
+
+    calculate_btn.click(
+        calculate_steps,
+        inputs=[
+            dataset_name,
+            dataset_split,
+            dataset_config,
+            tokenizer_name,
+            num_gpus_per_node,
+            num_nodes,
+            batch_size,
+            grad_accum,
+            block_size,
+            text_column,
+            token,
+        ],
+        outputs=[samples, steps],
+        api_name="calculate-training-steps",
+    )
+
+
+if __name__ == "__main__":
+    demo.queue().launch()

requirements.txt

@@ -0,0 +1,3 @@
+datasets
+gradio
+transformers