Add tokenizer generator

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/livebook-dev/livebook:latest-cuda11.8
+FROM ghcr.io/livebook-dev/livebook:latest
 
 ENV LIVEBOOK_APP_SERVICE_NAME "🐳 Hugging Face - $SPACE_TITLE"
 ENV LIVEBOOK_APP_SERVICE_URL "https://huggingface.co/spaces/$SPACE_AUTHOR_NAME/$SPACE_REPO_NAME"
@@ -9,7 +9,12 @@ ENV LIVEBOOK_DATA_PATH "/data"
 ENV LIVEBOOK_PORT 7860
 
 EXPOSE 7860
+
 USER root
+
+RUN apt-get update && apt-get install -y python3 python3-pip python-is-python3
+RUN pip --no-cache-dir install transformers sentencepiece protobuf
+
 COPY public-apps/ /public-apps
 RUN mkdir -p /data
 RUN chmod 777 /data

README.md

@@ -1,10 +1,10 @@
 ---
-title: Livebook
-emoji: 📓
-colorFrom: pink
-colorTo: purple
+title: Bumblebee tools
+emoji: 🐝
+colorFrom: yellow
+colorTo: brown
 sdk: docker
 fullWidth: true
 ---
 
-You can install and run [Livebook](https://livebook.dev/) inside a Hugging Face Space. Here's [a tutorial](https://huggingface.co/docs/hub/spaces-sdks-docker-livebook) on how to do that.
+Tools for [elixir-nx/bumblebee](https://github.com/elixir-nx/bumblebee).

public-apps/tokenizer-generator.livemd

@@ -0,0 +1,153 @@
+<!-- livebook:{"app_settings":{"access_type":"public","auto_shutdown_ms":5000,"multi_session":true,"output_type":"rich","show_source":true,"slug":"tokenizer-generator"}} -->
+
+# Tokenizer generator
+
+```elixir
+Mix.install([
+  {:kino, "~> 0.10.0"},
+  {:req, "~> 0.4.3"}
+])
+```
+
+## Info
+
+```elixir
+Kino.Markdown.new("""
+## Background
+
+HuggingFace repositories store tokenizers in two flavours:
+
+  1. "slow tokenizer" - corresponds to a tokenizer implemented in Python
+     and stored as `tokenizer_config.json`
+
+  2. "fast tokenizers" - corresponds to a tokenizer implemented in Rust
+     and stored as `tokenizer.json`
+
+Many repositories only include files for 1., but the `transformers` library
+automatically converts "slow tokenizer" to "fast tokenizer" whenever possible.
+
+Bumblebee relies on the Rust bindings and therefore always requires the
+`tokenizer.json` file. This app generates that file for any repository with the
+"slow tokenizer".
+""")
+```
+
+## Generator
+
+```elixir
+Kino.Markdown.new("## Converter")
+```
+
+```elixir
+{version, 0} =
+  System.cmd("python", ["-c", "import transformers; print(transformers.__version__, end='')"])
+
+Kino.Markdown.new("""
+`tokenizers: #{version}`
+""")
+```
+
+```elixir
+repo_input = Kino.Input.text("HuggingFace repo")
+```
+
+```elixir
+repo = Kino.Input.read(repo_input)
+
+if repo == "" do
+  Kino.interrupt!(:normal, "Enter repository.")
+end
+```
+
+```elixir
+response =
+  Req.post!("https://huggingface.co/api/models/#{repo}/paths-info/main",
+    json: %{paths: ["tokenizer.json"]}
+  )
+
+case response do
+  %{status: 200, body: []} ->
+    :ok
+
+  %{status: 200, body: [%{"path" => "tokenizer.json"}]} ->
+    Kino.interrupt!(:error, "The tokenizer.json file already exist in the given repository.")
+
+  _ ->
+    Kino.interrupt!(:error, "The repository does not exist or requires authentication.")
+end
+```
+
+```elixir
+output_dir = Path.join(System.tmp_dir!(), repo)
+```
+
+````elixir
+script = """
+import sys
+from transformers import AutoTokenizer
+
+repo = sys.argv[1]
+output_dir = sys.argv[2]
+
+
+try:
+  tokenizer = AutoTokenizer.from_pretrained(repo)
+  assert tokenizer.is_fast
+  tokenizer.save_pretrained(output_dir)
+except Exception as error:
+  print(error)
+  exit(1)
+"""
+
+case System.cmd("python", ["-c", script, repo, output_dir]) do
+  {_, 0} ->
+    :ok
+
+  {output, _} ->
+    Kino.Markdown.new("""
+    ```
+    #{output}
+    ```
+    """)
+    |> Kino.render()
+
+    Kino.interrupt!(:error, "Tokenizer conversion failed.")
+end
+````
+
+```elixir
+tokenizer_path = Path.join(output_dir, "tokenizer.json")
+
+Kino.Download.new(
+  fn -> File.read!(tokenizer_path) end,
+  filename: "tokenizer.json",
+  label: "tokenizer.json"
+)
+```
+
+`````elixir
+Kino.Markdown.new("""
+### Next steps
+
+  1. Go to https://huggingface.co/#{repo}/upload/main.
+
+  2. Upload the `tokenizer.json` file.
+
+  3. Add the following description:
+
+      ````markdown
+      Generated with:
+
+      ```python
+      from transformers import AutoTokenizer
+
+      tokenizer = AutoTokenizer.from_pretrained("#{repo}")
+      assert tokenizer.is_fast
+      tokenizer.save_pretrained("...")
+      ```
+      ````
+
+  4. Submit the PR.
+
+""")
+`````

public-apps/welcome.livemd

@@ -1,46 +0,0 @@
-<!-- livebook:{"app_settings":{"access_type":"public","slug":"welcome"}} -->
-
-# Livebook <3 Hugging Face
-
-```elixir
-Mix.install([
-  {:kino, "~> 0.9"}
-])
-```
-
-## Section
-
-This is the source of a deployed notebook.
-This notebook is static and simply renders the markdown content below.
-
-```elixir
-Kino.Markdown.new("""
-Welcome to Livebook in Hugging Face!
-
-This is a deployed notebook, which is also a perfect place to teach you
-the ropes in using Livebook with Hugging Face.
-
-## Getting started
-
-First off, if you want to run your own copy of Livebook,
-[check our tutorial](https://news.livebook.dev/livebook-inside-hugging-face-spaces-3LQaRi).
-Once you clone the space, remember to set `LIVEBOOK_PASSWORD` as
-an environment variable on your Space Settings page (a minimum of
-12 digits is required).
-
-If you are new to Elixir and Livebook, [head out to the Learn page](/learn)
-(it requires a password), there you will find resources to get started
-with both.
-
-## Deploying notebooks
-
-Livebook is fully collaborative and it enables you to deploy interactive
-and collaborative apps just as well. All of your deployable notebooks will
-be in the "public-apps" directory of your Spaces repository.
-
-To deploy your own notebook on Hugging Face, you must click the
-<i class="ri-livebook-deploy"></i> icon on the notebook sidebar, set a "Slug"
-for the notebook, mark it as public and then drop its `.livemd` file into
-the "public-apps" directory of your Spaces repo.
-""")
-```