Upload first working version

README.md

@@ -1,12 +1,6 @@
 ---
 title: Subspace
-emoji: 🐠
-colorFrom: red
-colorTo: yellow
+app_file: main.py
 sdk: gradio
 sdk_version: 3.35.2
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

main.py

@@ -0,0 +1,107 @@
+import gradio as gr
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pandas as pd
+import os
+
+os.environ['CURL_CA_BUNDLE'] = ''
+
+# Load dataset
+issues_dataset = load_dataset("gvozdev/subspace-info-v2", split="train")
+
+# Load tokenizer and model
+model_ckpt = "sentence-transformers/all-MiniLM-L12-v1"
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+model = AutoModel.from_pretrained(model_ckpt, trust_remote_code=True)
+
+# Text concatenation - not used in this case as mapping only on subject returns better results
+# def concatenate_text(examples):
+#     return {
+#         "text": examples["subject"]
+#         + " \n "
+#         + examples["details"]
+#     }
+
+
+issues_dataset = issues_dataset.map()
+
+# To speed up embedding, we can switch to GPU (change device to "cuda") - for larger models
+device = torch.device("cpu")
+model.to(device)
+
+
+# CLS pooling on model’s outputs: collect the last hidden state for the special [CLS] token
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+
+
+# Tokenize a list of documents, place the tensors on the CPU/GPU, feed them to the model,
+# and apply CLS pooling to the outputs
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="pt"
+    )
+    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+# Test if the  function works
+# embedding = get_embeddings(issues_dataset["details"][0])
+# print(embedding.shape)
+
+
+# Use Dataset.map() to apply get_embeddings() function to each row in the dataset and create a new "embeddings" column
+# Convert the embeddings to NumPy arrays as Datasets requires this format when we try to index them with FAISS
+embeddings_dataset = issues_dataset.map(
+    lambda x: {"embeddings": get_embeddings(x["subject"]).detach().cpu().numpy()[0]}
+)
+
+# Create a FAISS index
+embeddings_dataset.add_faiss_index(column="embeddings")
+
+
+#
+def answer_question(question):
+    # Get an embedding for the question
+    question_embedding = get_embeddings([question]).cpu().detach().numpy()
+
+    # Find a nearest neighbor in our dataset
+    scores, samples = embeddings_dataset.get_nearest_examples(
+        "embeddings", question_embedding, k=1
+    )
+
+    samples_df = pd.DataFrame.from_dict(samples)
+
+    # This part is needed in case we use k>1
+    # samples_df["scores"] = scores
+    # samples_df.sort_values("scores", ascending=False, inplace=True)
+
+    return samples_df["details"].values[0]
+
+
+# Gradio interface
+title = "Subspace Docs bot"
+description = '<p style="text-align: center;">This is a bot trained on Subspace Network documentation ' \
+              'to answer the most common questions about the project</p>'
+
+
+def chat(message, history):
+    history = history or []
+    response = answer_question(message)
+    history.append((message, response))
+    return history, history
+
+
+iface = gr.Interface(
+    chat,
+    ["text", "state"],
+    ["chatbot", "state"],
+    allow_flagging="never",
+    title=title,
+    description=description,
+    theme="Monochrome",
+    examples=["What is Subspace Network?", "Do you have a token?", "System requirements"]
+)
+
+iface.launch(share=True)

requirements.txt

@@ -0,0 +1,81 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.0
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.1.0
+cycler==0.11.0
+datasets==2.13.1
+dill==0.3.6
+einops==0.6.1
+evaluate==0.4.0
+exceptiongroup==1.1.1
+faiss-cpu==1.7.4
+fastapi==0.98.0
+ffmpy==0.3.0
+filelock==3.12.2
+fonttools==4.40.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+gradio==3.35.2
+gradio_client==0.2.7
+h11==0.14.0
+httpcore==0.17.2
+httpx==0.24.1
+huggingface-hub==0.15.1
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.3
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.25.0
+orjson==3.9.1
+packaging==23.1
+pandas==2.0.3
+Pillow==9.5.0
+pyarrow==12.0.1
+pydantic==1.10.9
+pydub==0.25.1
+Pygments==2.15.1
+pyparsing==3.1.0
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+regex==2023.6.3
+requests==2.27.1
+responses==0.18.0
+safetensors==0.3.1
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+sympy==1.12
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.1
+tqdm==4.65.0
+transformers==4.30.2
+typing_extensions==4.7.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==2.0.3
+uvicorn==0.22.0
+websockets==11.0.3
+xxhash==3.2.0
+yarl==1.9.2