Duplicate from justest/embeddings-api

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+/flagged

README.md

@@ -0,0 +1,14 @@
+---
+title: Embeddings Api
+emoji: 🏢
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 3.36.1
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: justest/embeddings-api
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,8 @@
+import gradio as gr
+import model
+
+def encode(text):
+    return model.encode(text)
+
+iface = gr.Interface(fn=encode, inputs="text", outputs="text")
+iface.launch()

init_data.py

@@ -0,0 +1,39 @@
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Distance, VectorParams
+from qdrant_client.http.models import PointStruct
+import tqdm
+import glob
+import model
+import re
+
+if __name__ == '__main__':
+    client = QdrantClient("127.0.0.1", port=6333)
+    collection_name = "mdn-docs"
+    client.recreate_collection(
+        collection_name=collection_name,
+        vectors_config=VectorParams(size=768, distance=Distance.COSINE),
+    )
+
+    count = 0
+    files = glob.glob("translated-content/files/zh-cn/**/*.md", recursive=True)
+    print(len(files))
+    for file in tqdm.tqdm(files):
+        count+=1
+        with open(file, 'r', encoding='utf-8') as f:
+            print('file', file)
+            text = f.read()
+            matchObj = re.match(r'\s*---[\n\r]+title:(((?!---).)+)', text, re.M|re.I)
+            if matchObj:
+                title = matchObj.group(1).strip()
+            else:
+                title = file
+
+            vector = model.encode(text)
+            client.upsert(
+                collection_name=collection_name,
+                wait=True,
+                points=[
+                    PointStruct(id=count, vector=vector, payload={"title": title, "text": text }),
+                ],
+            )
+

model.py

@@ -0,0 +1,7 @@
+from sentence_transformers import SentenceTransformer
+model = SentenceTransformer('moka-ai/m3e-base')
+# model = SentenceTransformer('nghuyong/ernie-3.0-base-zh')
+
+def encode(text: str):
+    text = text.replace("\n", " ")
+    return model.encode([text], normalize_embeddings=True).tolist()[0]

requirements.txt

@@ -0,0 +1 @@
+sentence-transformers==2.2.2

translated-content

@@ -0,0 +1 @@
+Subproject commit 79462bd3fd2533e3b71a117d1c98fafb8d4ca0e2