Update app.py

app.py

@@ -1,5 +1,118 @@
+# process.py
+from huggingface_hub import snapshot_download, HfApi
+from tokenizers import Tokenizer
+import os
+import json
+import threading
 import requests
+import pandas as pd
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
 
-r = requests.get("https://storage.googleapis.com/kagglesdsdata/datasets/10104711/15766657/tokenizer.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20260426%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20260426T150318Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=a2e4065f0b4e848ce90bf17823fc5740a0b2679dc1206f2f6e5dcb9344c173b00b01fdc1387b72469ae6a380afd200761d3dc4c7b412cb690c08deed4df18dd0329636cd1a76f4e2ce29b525878fcb6e33b26bf83d38caec23da8bdd717d20f7dd7659038bda8f4a4d5e707bbe6864cbd5a7bad5e3b16fdc7960d25e89fe6e36c033ab185e4ebea700de8b4e2ed2202340fa78cc741aa8a5d3aa51e02a2391329d4e15a27f270fed8a8efa3e6312b5a6363dc2250f80397bca6fd330d884de96a9fce60e848c427af37ff3df417c6a6ffbbd1ad15f796097f9a940da80c05720be29d8be055130a99b5e97f9a1f8ff9968a1970a931ca591978686bc3a2df3c9")
-with open("/data/tokenizer.json", "wb") as f:
-    f.write(r.content)
+# ── Config ──
+HF_USERNAME  = "Neon-coding"
+DATASET_NAME = "github-code-raw"
+RAW_DIR      = "/data/codeparrot-raw"
+OUT_DIR      = "/data/by-language"
+STATE_FILE   = "/data/progress_state.json"
+TOK_PATH     = "/data/tokenizer.json"
+
+os.makedirs(OUT_DIR, exist_ok=True)
+os.makedirs(RAW_DIR, exist_ok=True)
+
+# ── Load tokenizer (already in bucket) ──
+print("✓ Loading tokenizer from bucket...")
+tokenizer = Tokenizer.from_file(TOK_PATH)
+SEP_TOKEN = tokenizer.token_to_id("<eos>")
+print(f"✓ Tokenizer loaded | vocab: {tokenizer.get_vocab_size():,}")
+
+# ── Load state ──
+if os.path.exists(STATE_FILE):
+    with open(STATE_FILE) as f:
+        state = json.load(f)
+    print(f"Resuming — {len(state['processed_files'])} files already done")
+else:
+    state = {"processed_files": [], "lang_tokens": {}}
+    print("Starting fresh")
+
+lock = threading.Lock()
+
+def save_state():
+    with open(STATE_FILE, "w") as f:
+        json.dump(state, f, indent=2)
+
+# ── Download codeparrot ──
+print("\nDownloading codeparrot/github-code-clean...")
+local_dir = snapshot_download(
+    repo_id="codeparrot/github-code-clean",
+    repo_type="dataset",
+    local_dir=RAW_DIR,
+)
+
+parquet_files = sorted(Path(local_dir).rglob("*.parquet"))
+print(f"Found {len(parquet_files)} parquet files")
+
+# ── Process each file ──
+def process_file(path):
+    fname = str(path)
+
+    if fname in state["processed_files"]:
+        print(f"  SKIP {path.name}")
+        return
+
+    try:
+        df = pd.read_parquet(path)
+
+        for lang, group in df.groupby("language"):
+            lang_dir = os.path.join(OUT_DIR, lang)
+            os.makedirs(lang_dir, exist_ok=True)
+            out = os.path.join(lang_dir, f"{path.stem}.jsonl")
+
+            if os.path.exists(out):
+                continue
+
+            texts     = group["code"].dropna().tolist()
+            encoded   = tokenizer.encode_batch(texts)
+            tok_count = sum(len(e.ids) for e in encoded)
+
+            group[["code"]].rename(columns={"code": "text"}).to_json(
+                out, orient="records", lines=True
+            )
+
+            with lock:
+                state["lang_tokens"][lang] = state["lang_tokens"].get(lang, 0) + tok_count
+
+        with lock:
+            state["processed_files"].append(fname)
+            save_state()
+
+        print(f"  ✓ {path.name} | langs: {list(df['language'].unique())}")
+
+    except Exception as e:
+        print(f"  ✗ {path.name} ERROR: {e}")
+
+
+with ThreadPoolExecutor(max_workers=8) as ex:
+    list(ex.map(process_file, parquet_files))
+
+# ── Save per-language meta ──
+print("\nSaving per-language meta.json...")
+for lang, total_tokens in state["lang_tokens"].items():
+    lang_dir = os.path.join(OUT_DIR, lang)
+    os.makedirs(lang_dir, exist_ok=True)
+    with open(os.path.join(lang_dir, "meta.json"), "w") as f:
+        json.dump({"language": lang, "total_tokens": total_tokens}, f, indent=2)
+    print(f"  {lang}: {total_tokens:,}")
+
+# ── Push to HF ──
+print("\nPushing to HuggingFace...")
+api = HfApi()
+api.upload_folder(
+    folder_path=OUT_DIR,
+    repo_id=f"{HF_USERNAME}/{DATASET_NAME}",
+    repo_type="dataset",
+)
+
+print("\nDone!")
+for l, t in sorted(state["lang_tokens"].items(), key=lambda x: -x[1]):
+    print(f"  {l}: {t:,}")