Spaces:
Running
Running
Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os, threading, hashlib, re
|
| 2 |
+
from http.server import HTTPServer, BaseHTTPRequestHandler
|
| 3 |
+
from datasets import load_dataset, Dataset, DatasetDict
|
| 4 |
+
|
| 5 |
+
TARGET_REPO = "hybirdvaults/xalviq-knowledge-base"
|
| 6 |
+
SOURCES = [
|
| 7 |
+
("MBZUAI/Urdu-Instruct", "train", "urdu_instruct"),
|
| 8 |
+
("CohereForAI/aya_dataset", "train", "aya"),
|
| 9 |
+
("ai4bharat/indic-instruct-data-v0.1", "train", "indic_instruct"),
|
| 10 |
+
("Magpie-Align/Magpie-Pro-DPO-200K", "train", "magpie_dpo"),
|
| 11 |
+
("BAAI/COIG-CQIA", "train", "coig_cqia"),
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
os.environ["HUGGINGFACE_HUB_TOKEN"] = os.getenv("HF_TOKEN", "")
|
| 15 |
+
|
| 16 |
+
def clean_text(text):
|
| 17 |
+
if not isinstance(text, str): return ""
|
| 18 |
+
text = ''.join(ch for ch in text if ch.isprintable() or ch in '\n\r\t')
|
| 19 |
+
text = re.sub(r'\s+', ' ', text).strip()
|
| 20 |
+
return text
|
| 21 |
+
|
| 22 |
+
def process_dataset(source, split, name):
|
| 23 |
+
print(f"\n📥 Processing {name} from {source}")
|
| 24 |
+
ds = load_dataset(source, split=split)
|
| 25 |
+
def clean_sample(example):
|
| 26 |
+
text = example.get("text") or example.get("content") or example.get("instruction") or ""
|
| 27 |
+
return {"text": clean_text(text), "length": len(clean_text(text).split()), "source": name}
|
| 28 |
+
cleaned = ds.map(clean_sample, remove_columns=ds.column_names)
|
| 29 |
+
cleaned = cleaned.filter(lambda x: x["length"] > 50)
|
| 30 |
+
seen = set()
|
| 31 |
+
cleaned = cleaned.filter(lambda x: (h:=hashlib.sha256(x["text"].encode()).hexdigest()) not in seen and not seen.add(h))
|
| 32 |
+
print(f" ✅ Cleaned {len(cleaned)} samples")
|
| 33 |
+
return cleaned
|
| 34 |
+
|
| 35 |
+
def run_cleaning():
|
| 36 |
+
all_ds = {}
|
| 37 |
+
for src, split, name in SOURCES:
|
| 38 |
+
try:
|
| 39 |
+
all_ds[name] = process_dataset(src, split, name)
|
| 40 |
+
except Exception as e:
|
| 41 |
+
print(f"⚠️ Error {name}: {e}")
|
| 42 |
+
if all_ds:
|
| 43 |
+
DatasetDict(all_ds).push_to_hub(TARGET_REPO, private=False)
|
| 44 |
+
print(f"\n✅ Done! View at: https://huggingface.co/datasets/{TARGET_REPO}")
|
| 45 |
+
with open("/tmp/cleaning_done", "w") as f: f.write("done")
|
| 46 |
+
|
| 47 |
+
class Handler(BaseHTTPRequestHandler):
|
| 48 |
+
def do_GET(self):
|
| 49 |
+
self.send_response(200)
|
| 50 |
+
self.end_headers()
|
| 51 |
+
status = "Cleaning completed!" if os.path.exists("/tmp/cleaning_done") else "Cleaning in progress..."
|
| 52 |
+
self.wfile.write(f"<html><body><h1>{status}</h1><p><a href='https://huggingface.co/datasets/{TARGET_REPO}'>{TARGET_REPO}</a></p></body></html>".encode())
|
| 53 |
+
|
| 54 |
+
def start_server():
|
| 55 |
+
HTTPServer(('0.0.0.0', 7860), Handler).serve_forever()
|
| 56 |
+
|
| 57 |
+
if __name__ == "__main__":
|
| 58 |
+
threading.Thread(target=run_cleaning).start()
|
| 59 |
+
start_server()
|