Upload 7 files

app.py

@@ -19,7 +19,7 @@ completion_tfidf_vectorizer = joblib.load('completion-vectorizer.pkl')
 completion_tfidf_matrix = joblib.load('completion-tfidf-matrix.pkl')
 
 hub_token = os.environ.get("HUB_TOKEN")
-model_id = "nicholasKluge/TeenyTinyLlama-160m-Chat"
+model_id = "nicholasKluge/TeenyTinyLlama-460m-Chat"
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 
 
 model = AutoModelForCausalLM.from_pretrained(model_id, token=hub_token)
@@ -31,7 +31,7 @@ model.to(device)
 intro = """
 O TeenyTinyLlama é um modelo de linguagem compacto baseado na arquitetura Llama 2 ([TinyLlama implementation](https://huggingface.co/TinyLlama)).Esse modelo foi projetado para oferecer recursos eficientes de processamento de linguagem natural e, ao mesmo tempo, consumir poucos recursos. Esses modelos foram treinados aproveitando as [leis de escalonamento](https://arxiv.org/abs/2203.15556) para determinar o número ideal de tokens por parâmetro e incorporando o [pré-treinamento de preferências](https://arxiv.org/abs/2112.00861).
 
-Esse repositório contém uma versão de [TeenyTinyLlama-160m](https://huggingface.co/nicholasKluge/TeenyTinyLlama-160m) (`TeenyTinyLlama-160m-Chat`) afinada no [Instruct-Aira Dataset version 2.0](https://huggingface.co/datasets/nicholasKluge/instruct-aira-dataset-v2).
+Esse repositório contém uma versão de [TeenyTinyLlama-460m](https://huggingface.co/nicholasKluge/TeenyTinyLlama-460m) (`TeenyTinyLlama-460m-Chat`) afinada no [Instruct-Aira Dataset version 2.0](https://huggingface.co/datasets/nicholasKluge/instruct-aira-dataset-v2).
 
 ## Limitações
 
@@ -68,7 +68,7 @@ Se desejar apresentar uma reclamação sobre qualquer mensagem produzida, por fa
 
 with gr.Blocks(theme='freddyaboulton/dracula_revamped') as demo:
 
-    gr.Markdown("""<h1><center>TeenyTinyLlama-160m-Chat 🦙💬</h1></center>""")
+    gr.Markdown("""<h1><center>TeenyTinyLlama-Chat 🦙💬</h1></center>""")
     gr.Markdown(intro)
 
     

completion-vectorizer.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64bbd49731efda1677dca239d838058ef221de4770ed5663e451af1347c30637
+oid sha256:add850bde149e5de855d3c0334cd99ef5055289f8d103626250db2b5a1bbd0dc
 size 4036115

completion_tfidf-matrix.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16f7cb342da64a6334bb035d162a29579853926af2243c14029fb5043d4fbd81
+size 116328867

create-tfidf-matrix.py

@@ -0,0 +1,46 @@
+import joblib
+import pandas as pd
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.feature_extraction.text import TfidfVectorizer
+import argparse
+
+def main():
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser.add_argument('--input', type=str, help="Input file path (file should be in parquet format and have 'prompt' and 'completion' columns)")
+    parser.add_argument('--output', type=str, help='Output file path')
+    args = parser.parse_args()
+
+    df = pd.read_parquet(args.input)
+
+    # fit the vectorizer on the prompt column
+    prompt_tfidf_vectorizer = TfidfVectorizer()
+    prompt_tfidf_vectorizer.fit(df['prompt'])
+
+    # save the vectorizer
+    joblib.dump(prompt_tfidf_vectorizer, args.output + 'prompt-vectorizer.pkl')
+
+    # get the tfidf_matrix
+    prompt_tfidf_matrix = prompt_tfidf_vectorizer.transform(df['prompt'])
+
+    # save the tfidf_matrix
+    joblib.dump(prompt_tfidf_matrix, args.output + 'prompt-tfidf_matrix.pkl')
+
+    # fit the vectorizer on the completion column
+    completion_tfidf_vectorizer = TfidfVectorizer()
+    completion_tfidf_vectorizer.fit(df['completion'])
+
+    # save the vectorizer
+    joblib.dump(completion_tfidf_vectorizer, args.output + 'completion-vectorizer.pkl')
+
+    # get the tfidf_matrix
+    completion_tfidf_matrix = completion_tfidf_vectorizer.transform(df['completion'])
+
+    # save the tfidf_matrix
+    joblib.dump(completion_tfidf_matrix, args.output + 'completion_tfidf-matrix.pkl')
+
+    print("Done!")
+
+if __name__ == '__main__':
+    main()
+
+# example usage: python create-tfidf-matrix.py --input fine-tuning-data.parquet --output ./

fine-tuning-data.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0eb2d74b25cf773e7a2edbcdc3d05818e80eaa494d124e4deb5820158958d7c
-size 89839763
+oid sha256:ddb05796faee1bfad1857780ee76ce08655bbc44c8d1391325b6bddd638f9d99
+size 89835929

prompt-tfidf_matrix.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95d8c1d302b36e5fef3da79e802354972158b247051715c98d55f351b8993fe2
+size 37977659

prompt-vectorizer.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29a72f7e1c286ffe7c87c5384484f5471baec4b3a86238b458fdbcfb52d01a38
+oid sha256:739df119b82ef1f2d8dfd4d85bc1ee489d2705b48d1bd701627df9222e15cc8f
 size 3324940