Duplicate from huggingface-projects/sd-multiplayer-bot

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,21 @@
+.DS_Store
+node_modules
+/build
+/.svelte-kit
+/package
+.env
+.env.*
+!.env.example
+
+# Ignore files for PNPM, NPM and YARN
+pnpm-lock.yaml
+package-lock.json
+yarn.lock
+venv/
+__pycache__/
+flagged/
+data
+data.db
+data.json
+rooms_data.db
+sd-multiplayer-data/

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+# Git LFS
+RUN apt-get update && apt-get install -y git-lfs
+RUN git lfs install
+
+
+# User
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME /home/user
+ENV PATH $HOME/.local/bin:$PATH
+WORKDIR $HOME
+RUN mkdir app
+WORKDIR $HOME/app
+
+COPY . $HOME/app
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,12 @@
+---
+title: Sd Multiplayer Bot
+emoji: 🤖
+colorFrom: red
+colorTo: indigo
+sdk: docker
+app_port: 7860
+pinned: false
+duplicated_from: huggingface-projects/sd-multiplayer-bot
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,120 @@
+import boto3
+import os
+import re
+import json
+from pathlib import Path
+import sqlite3
+from huggingface_hub import Repository, HfFolder
+import tqdm
+import subprocess
+
+from fastapi import FastAPI
+from fastapi_utils.tasks import repeat_every
+
+
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_KEY = os.getenv('AWS_SECRET_KEY')
+AWS_S3_BUCKET_NAME = os.getenv('AWS_S3_BUCKET_NAME')
+
+s3 = boto3.client(service_name='s3',
+                  aws_access_key_id=AWS_ACCESS_KEY_ID,
+                  aws_secret_access_key=AWS_SECRET_KEY)
+
+
+paginator = s3.get_paginator('list_objects_v2')
+
+
+S3_DATA_FOLDER = Path("sd-multiplayer-data")
+ROOMS_DATA_DB = S3_DATA_FOLDER / "rooms_data.db"
+
+
+repo = Repository(
+    local_dir=S3_DATA_FOLDER,
+    repo_type="dataset",
+    clone_from="huggingface-projects/sd-multiplayer-data",
+    use_auth_token=True,
+)
+repo.git_pull()
+
+
+if not ROOMS_DATA_DB.exists():
+    print("Creating database")
+    print("ROOMS_DATA_DB", ROOMS_DATA_DB)
+    db = sqlite3.connect(ROOMS_DATA_DB)
+    with open(Path("schema.sql"), "r") as f:
+        db.executescript(f.read())
+    db.commit()
+    db.close()
+
+
+def get_db(db_path):
+    db = sqlite3.connect(db_path, check_same_thread=False)
+    db.row_factory = sqlite3.Row
+    try:
+        yield db
+    except Exception:
+        db.rollback()
+    finally:
+        db.close()
+
+
+def sync_rooms_to_dataset():
+    for room_data_db in get_db(ROOMS_DATA_DB):
+        rooms = room_data_db.execute("SELECT * FROM rooms").fetchall()
+        cursor = room_data_db.cursor()
+        for row in tqdm.tqdm(rooms):
+            room_id = row["room_id"]
+            print("syncing room data: ", room_id)
+
+            objects = []
+            for result in paginator.paginate(Bucket=AWS_S3_BUCKET_NAME, Prefix=f'{room_id}/', Delimiter='/'):
+                results = []
+                for obj in result.get('Contents'):
+                    try:
+                        key = obj.get('Key')
+                        time = obj.get('LastModified').isoformat()
+                        split_str = re.split(r'[-/.]', key)
+                        uuid = split_str[3]
+                        x, y = [int(n)
+                                for n in re.split(r'[_]', split_str[4])]
+                        prompt = ' '.join(split_str[4:])
+                        results.append(
+                            {'x': x, 'y': y, 'prompt': prompt, 'time': time, 'key': key, 'uuid': uuid})
+                        cursor.execute(
+                            'INSERT INTO rooms_data VALUES (NULL, ?, ?, ?, ?, ?, ?, ?)', (room_id, uuid, x, y, prompt, time, key))
+                    except Exception as e:
+                        print(e)
+                        continue
+
+                objects += results
+            room_data_db.commit()
+
+            all_rows = [dict(row) for row in room_data_db.execute(
+                "SELECT * FROM rooms_data WHERE room_id = ?", (room_id,)).fetchall()]
+            data_path = S3_DATA_FOLDER / f"{room_id}.json"
+            with open(data_path, 'w') as f:
+                json.dump(all_rows, f, separators=(',', ':'))
+    print("Updating repository")
+    subprocess.Popen(
+        "git add . && git commit --amend -m 'update' && git push --force", cwd=S3_DATA_FOLDER, shell=True)
+
+
+app = FastAPI()
+
+
+@app.get("/")
+def read_root():
+    return "Just a bot to sync data to huggingface datasets and tweet tha latest data"
+
+
+@app.get("/sync")
+def sync():
+    sync_rooms_to_dataset()
+    return "Synced data to huggingface datasets"
+
+
+@app.on_event("startup")
+@repeat_every(seconds=1800)
+def repeat_sync():
+    sync_rooms_to_dataset()
+    return "Synced data to huggingface datasets"

requirements.txt

@@ -0,0 +1,6 @@
+huggingface-hub==0.10
+fastapi-utils==0.2
+uvicorn==0.19
+tqdm==4.64
+boto3==1.26
+fastapi==0.86

schema.sql

@@ -0,0 +1,59 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE rooms (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    room_id TEXT NOT NULL
+);INSERT INTO rooms VALUES(1,'room-0');
+INSERT INTO rooms VALUES(2,'room-1');
+INSERT INTO rooms VALUES(3,'room-2');
+INSERT INTO rooms VALUES(4,'room-3');
+INSERT INTO rooms VALUES(5,'room-4');
+INSERT INTO rooms VALUES(6,'room-5');
+INSERT INTO rooms VALUES(7,'room-6');
+INSERT INTO rooms VALUES(8,'room-7');
+INSERT INTO rooms VALUES(9,'room-8');
+INSERT INTO rooms VALUES(10,'room-9');
+INSERT INTO rooms VALUES(11,'room-10');
+INSERT INTO rooms VALUES(12,'room-11');
+INSERT INTO rooms VALUES(13,'room-12');
+INSERT INTO rooms VALUES(14,'room-13');
+INSERT INTO rooms VALUES(15,'room-14');
+INSERT INTO rooms VALUES(16,'room-15');
+INSERT INTO rooms VALUES(17,'room-16');
+INSERT INTO rooms VALUES(18,'room-17');
+INSERT INTO rooms VALUES(19,'room-18');
+INSERT INTO rooms VALUES(20,'room-19');
+INSERT INTO rooms VALUES(21,'room-20');
+INSERT INTO rooms VALUES(22,'room-21');
+INSERT INTO rooms VALUES(23,'room-22');
+INSERT INTO rooms VALUES(24,'room-23');
+INSERT INTO rooms VALUES(25,'room-24');
+INSERT INTO rooms VALUES(26,'room-25');
+INSERT INTO rooms VALUES(27,'room-26');
+INSERT INTO rooms VALUES(28,'room-27');
+INSERT INTO rooms VALUES(29,'room-28');
+INSERT INTO rooms VALUES(30,'room-29');
+INSERT INTO rooms VALUES(31,'room-30');
+INSERT INTO rooms VALUES(32,'room-31');
+INSERT INTO rooms VALUES(33,'room-32');
+INSERT INTO rooms VALUES(34,'room-33');
+INSERT INTO rooms VALUES(35,'room-34');
+INSERT INTO rooms VALUES(36,'room-35');
+INSERT INTO rooms VALUES(37,'room-36');
+INSERT INTO rooms VALUES(38,'room-37');
+INSERT INTO rooms VALUES(39,'room-38');
+INSERT INTO rooms VALUES(40,'room-39');
+INSERT INTO rooms VALUES(41,'room-40');
+DELETE FROM sqlite_sequence;
+INSERT INTO sqlite_sequence VALUES('rooms',41);
+CREATE TABLE rooms_data (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    room_id TEXT NOT NULL,
+    uuid TEXT NOT NULL,
+    x INTEGER NOT NULL,
+    y INTEGER NOT NULL,
+    prompt TEXT NOT NULL,
+    time DATETIME NOT NULL,
+    key TEXT NOT NULL,
+    UNIQUE (key) ON CONFLICT IGNORE
+);COMMIT;