initial_commit

Dockerfile

@@ -0,0 +1,49 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Install git as root
+RUN apt-get update && \
+    apt-get install -y wget gnupg nmap && \
+    # Add the LLVM repository for clang
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-15 main" > /etc/apt/sources.list.d/llvm-toolchain.list && \
+    apt-get update && \
+    apt-get install -y git cmake clang && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create a non-root user (do this as root)
+RUN useradd -ms /bin/bash myuser
+
+# Set the working directory as root before switching users
+WORKDIR /home/myuser/app
+
+# Copy files and change permissions as root
+COPY requirements.txt /home/myuser/app/requirements.txt
+COPY setup.sh /home/myuser/app/setup.sh
+RUN chmod +x /home/myuser/app/setup.sh
+
+# Install Python dependencies as root
+RUN pip install --no-cache-dir -r /home/myuser/app/requirements.txt
+
+# Run setup.sh as root (if it needs elevated privileges)
+RUN /home/myuser/app/setup.sh
+
+# Change ownership of the directory to myuser
+RUN chown -R myuser:myuser /home/myuser/app
+
+# Switch to the non-root user
+USER myuser
+
+
+RUN ls /home/myuser/app
+# Copy the rest of the application code to the container (as myuser)
+COPY . /home/myuser/app
+
+# Expose the necessary port
+EXPOSE 7860
+
+# Set environment variable for Gradio
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+
+# Command to run the application
+CMD ["python", "app.py"]

README.md

@@ -1,10 +1,22 @@
 ---
 title: BitNet.cpp
-emoji: 🌖
+emoji: 💻
 colorFrom: blue
-colorTo: blue
-sdk: docker
+colorTo: red
+sdk: gradio
+sdk_version: 4.27.0
+app_file: app.py
 pinned: false
+
+hf_oauth: true
+# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
+hf_oauth_expiration_minutes: 480
+# optional, see "Scopes" below. "openid profile" is always included.
+hf_oauth_scopes:
+ - read-repos
+ - write-repos
+ - manage-repos
+ - inference-api
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,135 @@
+import gradio as gr
+import subprocess
+import os
+import time
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+
+# Path to the cloned repository
+BITNET_REPO_PATH = "/home/myuser/app/BitNet"
+SETUP_SCRIPT = os.path.join(BITNET_REPO_PATH, "setup_env.py")
+INFERENCE_SCRIPT = os.path.join(BITNET_REPO_PATH, "run_inference.py")
+
+# Function to set up the environment by running setup.py
+def setup_bitnet(model_name):
+    try:
+        result = subprocess.run(
+            f"python {SETUP_SCRIPT} --hf-repo {model_name} -q i2_s",
+            shell=True,
+            cwd=BITNET_REPO_PATH,
+            capture_output=True,
+            text=True
+        )
+        if result.returncode == 0:
+            return "Setup completed successfully!"
+        else:
+            return f"Error in setup: {result.stderr}"
+    except Exception as e:
+        return str(e)
+
+# Function to run inference using the `run_inference.py` file
+def run_inference(model_name, input_text, num_tokens=6):
+    try:
+        # Call the `run_inference.py` script with the model and input
+        start_time = time.time()
+        result = subprocess.run(
+            f"python run_inference.py -m models/Llama3-8B-1.58-100B-tokens/ggml-model-i2_s.gguf -p \"{input_text}\" -n {num_tokens} -temp 0",
+            shell=True,
+            cwd=BITNET_REPO_PATH,
+            capture_output=True,
+            text=True
+        )
+        end_time = time.time()
+        
+        if result.returncode == 0:
+            inference_time = round(end_time - start_time, 2)
+            return result.stdout, f"Inference took {inference_time} seconds."
+        else:
+            return f"Error during inference: {result.stderr}", None
+    except Exception as e:
+        return str(e), None
+
+def run_transformers(profile: gr.OAuthProfile | None, oauth_token: gr.OAuthToken | None, model_name, input_text, num_tokens):
+
+    if oauth_token is None : 
+        return "Error : To Compare please login to your HF account and make sure you have access to the used Llama models"
+    # Load the model and tokenizer dynamically if needed (commented out for performance)
+    tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=oauth_token.token)
+    model = AutoModelForCausalLM.from_pretrained(model_name, use_auth_token=oauth_token.token)
+
+    # Encode the input text
+    input_ids = tokenizer.encode(input_text, return_tensors="pt")
+
+    # Start time for inference
+    start_time = time.time()
+
+    # Generate output with the specified number of tokens
+    output = model.generate(input_ids, max_length=len(input_ids[0]) + num_tokens, num_return_sequences=1)
+
+    # Calculate inference time
+    inference_time = time.time() - start_time
+
+    # Decode the generated output
+    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
+
+    return generated_text, f"{inference_time:.2f} seconds"
+
+# Gradio Interface
+def interface():
+    with gr.Blocks(css=".gr-button {background-color: #5C6BC0; color: white;} .gr-button:hover {background-color: #3F51B5;}") as demo:
+
+        gr.LoginButton(elem_id="login-button", elem_classes="center-button")
+
+        gr.Markdown(
+            """
+            <h1 style="text-align: center; color: #4A148C;">BitNet.cpp Speed Demonstration</h1>
+            <p style="text-align: center; color: #6A1B9A;">Compare the speed and performance of BitNet with Transformers!</p>
+            """, 
+            elem_id="header"
+        )
+
+        # Model selection and setup row
+        with gr.Row():
+            model_dropdown = gr.Dropdown(
+                label="Select Model",
+                choices=["HF1BitLLM/Llama3-8B-1.58-100B-tokens", "1bitLLM/bitnet_b1_58-3B", "1bitLLM/bitnet_b1_58-large"],  # Replace with available models
+                value="HF1BitLLM/Llama3-8B-1.58-100B-tokens",
+                interactive=True,
+                elem_id="model-dropdown"
+            )
+            setup_button = gr.Button("Run Setup", elem_id="setup-button")
+            setup_status = gr.Textbox(label="Setup Status", interactive=False, placeholder="Setup status will appear here...")
+
+        # Inference row
+        with gr.Row():
+            num_tokens = gr.Slider(minimum=1, maximum=100, label="Number of Tokens to Generate", value=50, step=1)
+            input_text = gr.Textbox(label="Input Text", placeholder="Enter your input text here...")
+            infer_button = gr.Button("Run Inference", elem_id="infer-button")
+            result_output = gr.Textbox(label="Output", interactive=False, placeholder="Inference output will appear here...")
+            time_output = gr.Textbox(label="Inference Time", interactive=False, placeholder="Inference time will appear here...")
+
+        # Comparison with Transformers
+        with gr.Row():
+            transformer_model_dropdown = gr.Dropdown(
+                label="Select Transformers Model",
+                choices=["meta-llama/Llama-3.1-8B", "meta-llama/Llama-3.2-3B", "meta-llama/Llama-3.2-1B"],  # Replace with actual models
+                value="meta-llama/Llama-3.1-8B",
+                interactive=True
+            )
+            compare_button = gr.Button("Run Transformers Inference", elem_id="compare-button")
+            transformer_result_output = gr.Textbox(label="Transformers Output", interactive=False, placeholder="Transformers output will appear here...")
+            transformer_time_output = gr.Textbox(label="Transformers Inference Time", interactive=False, placeholder="Transformers inference time will appear here...")
+
+        # Actions
+        setup_button.click(setup_bitnet, inputs=model_dropdown, outputs=setup_status)
+        infer_button.click(run_inference, inputs=[model_dropdown, input_text, num_tokens], outputs=[result_output, time_output])
+        compare_button.click(run_transformers, inputs=[transformer_model_dropdown, input_text, num_tokens], outputs=[transformer_result_output, transformer_time_output])
+
+# Launch the Gradio app    
+    return demo
+
+demo = interface()
+demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+https://gradio-builds.s3.amazonaws.com/4485dd46a8e4b3f5b35e42d52f291b72fdc1a952/gradio-4.39.0-py3-none-any.whl
+pydantic==2.8.2
+pydantic-core==2.20.1
+fastapi==0.112.4
+huggingface-hub
+transformers

setup.sh

@@ -0,0 +1,2 @@
+git clone --recursive https://github.com/microsoft/BitNet.git
+pip install -r BitNet/requirements.txt