Duplicate from narugo/jupyterlab_crawl_nikke

.bashrc

@@ -0,0 +1 @@
+export PATH=$HOME/.local/bin:$PATH

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+venv

Dockerfile

@@ -0,0 +1,41 @@
+FROM python:3.8.1
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN apt-get update && \
+      apt-get install -y sudo tmux wget curl htop make tree nano && \
+      apt-get install -y iputils-ping telnet && \
+      apt-get install -y ffmpeg libsm6 libxext6 && \
+      apt-get install -y git git-lfs
+
+RUN --mount=type=secret,id=PASSWORD,mode=0444,required=true \
+      useradd -m -u 1000 user && \
+      echo "user:$(cat /run/secrets/PASSWORD)" | chpasswd && \
+      adduser user sudo
+
+RUN pip install -U pip pysocks
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+USER user
+ENV HOME=/home/user
+ENV PATH=$HOME/.local/bin:$PATH
+ENV SHELL=/bin/bash
+
+WORKDIR $HOME
+
+COPY --chown=user . $HOME/app
+
+COPY .bashrc $HOME/.bashrc_append
+RUN cat $HOME/.bashrc_append >> $HOME/.bashrc && \
+      rm $HOME/.bashrc_append
+
+RUN git clone https://github.com/deepghs/cyberharem.git $HOME/cyberharem && \
+      pip install -r $HOME/cyberharem/requirements.txt && \
+      pip install onnxruntime
+RUN cp $HOME/app/test_crawl.py $HOME/cyberharem
+
+EXPOSE     7860
+ENTRYPOINT []
+CMD        ["/bin/bash", "./app/run.sh"]

README.md

@@ -0,0 +1,13 @@
+---
+title: JupyterLab
+emoji: 💹
+colorFrom: blue
+colorTo: red
+sdk: docker
+pinned: false
+license: mit
+app_port: 7860
+duplicated_from: narugo/jupyterlab_crawl_nikke
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,3 @@
+jupyterlab==3.6.1
+jupyter-server==2.3.0
+tornado==6.2

run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+PASSWORD="${PASSWORD:=huggingface}"
+echo "Starting Jupyter Lab with token $PASSWORD"
+
+jupyter lab \
+    --ip=0.0.0.0 \
+    --port=7860 \
+    --no-browser \
+    --allow-root \
+    --NotebookApp.token=$PASSWORD \
+    --NotebookApp.tornado_settings="{'headers': {'Content-Security-Policy': 'frame-ancestors *'}}" \
+    --NotebookApp.cookie_options="{'SameSite': 'None', 'Secure': True}" \
+    --NotebookApp.disable_check_xsrf=True
+

test_crawl.py

@@ -0,0 +1,66 @@
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+from ditk import logging
+from gchar.games.azurlane import Character
+from gchar.resources.pixiv import get_pixiv_posts
+from tqdm.auto import tqdm
+from waifuc.utils import task_ctx
+
+from cyberharem.dataset import crawl_dataset_to_huggingface
+from cyberharem.utils import get_hf_fs
+
+
+def _cht(ch: Character):
+    name = str(ch.enname or ch.cnname or ch.jpname)
+    short_name = re.sub(r'[\W_]+', '_', name).strip('_')
+    return f'{short_name}_{ch.__game_name__}'
+
+
+def _get_pixiv_posts(ch: Character):
+    v = get_pixiv_posts(ch)
+    return 0 if v is None else v[0]
+
+
+hf_fs = get_hf_fs()
+# all_chs = [ch for ch in Character.all(contains_extra=False) if not ch.is_extra and ch.accessible and ch.index >= 153]
+all_chs = Character.all(contains_extra=False)
+all_chs = sorted(all_chs, key=lambda x: -_get_pixiv_posts(x))[2::3]
+pg = tqdm(total=len(all_chs))
+crawled_ids = set()
+
+
+
+def _crawl(char_):
+    try:
+        repo = f'CyberHarem/{_cht(char_)}'
+        if char_.gender == 'female':
+            if char_.index not in crawled_ids and \
+                    not hf_fs.exists(f'datasets/{repo}/dataset-raw.zip'):
+                with task_ctx(repo):
+                    crawl_dataset_to_huggingface(
+                        char_, repository=repo, limit=200
+                    )
+                    crawled_ids.add(char_.index)
+                    pg.update()
+            else:
+                logging.info(f'{char_!r} already crawled, skipped.')
+                pg.update()
+        else:
+            logging.info(f'{char_!r} is not female, skipped!')
+            pg.update()
+    except Exception as err:
+        logging.error(repr(err))
+        raise
+
+
+if __name__ == '__main__':
+    logging.try_init_root(logging.INFO)
+    tp = ThreadPoolExecutor(max_workers=1)
+
+    for ch in all_chs:
+        # _crawl(ch)
+        tp.submit(_crawl, ch)
+
+    tp.shutdown()
+