Upload 8 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+en-other_tab.txt filter=lfs diff=lfs merge=lfs -text
+ref.txt filter=lfs diff=lfs merge=lfs -text
+src.txt filter=lfs diff=lfs merge=lfs -text
+trg.txt filter=lfs diff=lfs merge=lfs -text

distillation.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Jun 17 16:20:22 2023
+
+@author: fujidai
+"""
+
+
+from sentence_transformers import SentenceTransformer, LoggingHandler, models, evaluation, losses
+import torch
+from torch.utils.data import DataLoader
+from sentence_transformers.datasets import ParallelSentencesDataset
+from datetime import datetime
+
+import os
+import logging
+import sentence_transformers.util
+import csv
+import gzip
+from tqdm.autonotebook import tqdm
+import numpy as np
+import zipfile
+import io
+
+logging.basicConfig(format='%(asctime)s - %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S',
+                    level=logging.INFO,
+                    handlers=[LoggingHandler()])
+logger = logging.getLogger(__name__)
+
+
+teacher_model_name = 'metrics_finetuning_teacher.py で作成した教師モデル'   #Our monolingual teacher model, we want to convert to multiple languages
+
+student_model_name = 'metrics_finetuning_student.py　で作成した生徒モデル'       #Multilingual base model we use to imitate the teacher model
+
+
+max_seq_length = 128                #Student model max. lengths for inputs (number of word pieces)
+train_batch_size = 64               #Batch size for training
+inference_batch_size = 64           #Batch size at inference
+max_sentences_per_language = 500000 #Maximum number of  parallel sentences for training
+train_max_sentence_length = 250     #Maximum length (characters) for parallel training sentences
+
+num_epochs = 100                       #Train for x epochs
+num_warmup_steps = 10000             #Warumup steps
+
+num_evaluation_steps = 1000          #Evaluate performance after every xxxx steps
+dev_sentences = 1000                 #Number of parallel sentences to be used for development
+
+
+######## Start the extension of the teacher model to multiple languages ########
+logger.info("Load teacher model")
+teacher_model = SentenceTransformer(teacher_model_name,device='mps')
+
+
+logger.info("Create student model from scratch")
+
+word_embedding_model = models.Transformer(student_model_name, max_seq_length=max_seq_length)
+# Apply mean pooling to get one fixed sized sentence vector
+pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())#denseで次元数を７６８にする次元数をいじる
+student_model = SentenceTransformer(modules=[word_embedding_model, pooling_model],device='mps')
+
+print(teacher_model)
+print(student_model)
+
+
+from sentence_transformers.datasets import ParallelSentencesDataset
+
+train_data = ParallelSentencesDataset(student_model=student_model, teacher_model=teacher_model)
+train_data.load_data('/en-other_tab.txt')#　英語　タブ　他の言語　というようになっている文
+
+
+#train_data.load_data('/Users/fujidai/TED2020_data/data/tuikazumi/en-ja/TED2020.en-ja.en')
+train_dataloader = DataLoader(train_data, shuffle=True, batch_size=train_batch_size)
+train_loss = losses.MSELoss(model=student_model)
+
+print(train_data)
+
+
+#50000_all-MiniLM-L6-v2__paraphrase-distilroberta-base-v2_epoch-1
+
+# Train the model
+print('az')
+student_model.fit(train_objectives=[(train_dataloader, train_loss)],
+          epochs=num_epochs,
+          #device=device,
+          warmup_steps=num_warmup_steps,
+          evaluation_steps=num_evaluation_steps,
+          optimizer_params= {'lr': 2e-5, 'eps': 1e-6},
+          checkpoint_path='checkpoint-savename',
+          checkpoint_save_steps=2000#その時に応じて変更する
+          )
+
+student_model.save('savename')
+
+
+
+
+
+
+
+
+
+
+
+#

en-other_tab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd96b89dfd2f0fdb933772cd5aa11b2bf89ed86dfba9a2fd4563bfea22fe22e
+size 27508845

metrics_finetuning_student.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 17 14:12:16 2023
+
+@author: fujidai
+"""
+
+
+import torch
+from sentence_transformers import SentenceTransformer, InputExample, losses,models
+from sentence_transformers import SentenceTransformer,  SentencesDataset, LoggingHandler, losses
+from sentence_transformers.readers import InputExample
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer
+from sentence_transformers.SentenceTransformer import SentenceTransformer
+import torch
+import torch.nn.functional as F
+import numpy as np
+from sentence_transformers import SentenceTransformer, util
+
+
+word_embedding_model = models.Transformer('/paraphrase-multilingual-mpnet-base-v2', max_seq_length=512)# modelの指定をする
+
+pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
+#dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(),out_features=16)
+model = SentenceTransformer(modules=[word_embedding_model, pooling_model],device='mps')
+print(model)
+
+
+with open('/da_seikika.txt', 'r') as f:#Negative en-ja cos_sim
+
+    raberu = f.read()
+raberu_lines = raberu.splitlines()#改行コードごとにリストに入れている
+data = []
+for i in range(len(raberu_lines)):
+    data.append(float(raberu_lines[i]))#Negative en-ja cos_simをdataに入れている
+
+
+
+
+with open('/src.txt', 'r') as f:#TEDのenglish
+    left = f.read()
+left_lines = left.splitlines()
+
+
+with open('/trg.txt', 'r') as f:#pseudo japanese (TEDのenglishをgoogle翻訳に入れた疑似コーパス)
+    right = f.read()
+right_lines = right.splitlines()#改行コードごとにリストに入れている
+
+
+train_examples = []
+for i in range(len(left_lines)):
+    pair=[]
+    pair.append(left_lines[i])#left_lines側のi行目をtextsに追加している
+    pair.append(right_lines[i])#right_lines側のi行目をtextsに追加している
+
+    example = InputExample(texts=pair, label=data[i])#textsをラベル付きで追加している
+    #print(example)#
+    #label=1-data[i]の１は positive cos_sim
+    #if aq>=0.25:
+    train_examples.append(example)#学習として入れるものに入れている
+print(len(train_examples))
+
+
+device = torch.device('mps')
+#print(device)
+
+import torch.nn.functional as F
+
+
+train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=8)
+#train_loss = losses.MarginMSELoss(model=model,similarity_fct=F.cosine_similarity)
+train_loss = losses.CosineSimilarityLoss(model)
+
+
+#Tune the model
+model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=100, warmup_steps=100,show_progress_bar=True,
+          #output_path='完成2best-6-30',
+          checkpoint_path='checkpoint-savename',checkpoint_save_steps=6699,#どのくらいのイテレーションごとに保存するか
+          save_best_model=True#,#,#checkpoint_save_total_limit=5
+          #optimizer_params= {'lr': 2e-6}#
+
+          )
+model.save("savename")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#

metrics_finetuning_teacher.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 17 14:12:16 2023
+
+@author: fujidai
+"""
+
+
+import torch
+from sentence_transformers import SentenceTransformer, InputExample, losses,models
+from sentence_transformers import SentenceTransformer,  SentencesDataset, LoggingHandler, losses
+from sentence_transformers.readers import InputExample
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer
+from sentence_transformers.SentenceTransformer import SentenceTransformer
+import torch
+import torch.nn.functional as F
+import numpy as np
+from sentence_transformers import SentenceTransformer, util
+
+
+word_embedding_model = models.Transformer('paraphrase-mpnet-base-v2', max_seq_length=512)# modelの指定をする
+
+pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
+#dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(),out_features=16)
+model = SentenceTransformer(modules=[word_embedding_model, pooling_model],device='mps')
+print(model)
+
+
+with open('/da_seikika.txt', 'r') as f:#Negative en-ja cos_sim
+
+    raberu = f.read()
+raberu_lines = raberu.splitlines()#改行コードごとにリストに入れている
+data = []
+for i in range(len(raberu_lines)):
+    data.append(float(raberu_lines[i]))#Negative en-ja cos_simをdataに入れている
+
+
+
+
+with open('/ref.txt', 'r') as f:#TEDのenglish
+    left = f.read()
+left_lines = left.splitlines()
+
+
+with open('/trg.txt', 'r') as f:#pseudo japanese (TEDのenglishをgoogle翻訳に入れた疑似コーパス)
+    right = f.read()
+right_lines = right.splitlines()#改行コードごとにリストに入れている
+
+
+train_examples = []
+for i in range(len(left_lines)):
+    pair=[]
+    pair.append(left_lines[i])#left_lines側のi行目をtextsに追加している
+    pair.append(right_lines[i])#right_lines側のi行目をtextsに追加している
+
+    example = InputExample(texts=pair, label=data[i])#textsをラベル付きで追加している
+    #print(example)#
+    #label=1-data[i]の１は positive cos_sim
+    #if aq>=0.25:
+    train_examples.append(example)#学習として入れるものに入れている
+print(len(train_examples))
+
+
+device = torch.device('mps')
+#print(device)
+
+import torch.nn.functional as F
+
+
+train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=8)
+#train_loss = losses.MarginMSELoss(model=model,similarity_fct=F.cosine_similarity)
+train_loss = losses.CosineSimilarityLoss(model)
+
+
+#Tune the model
+model.fit(train_objectives=[(train_dataloader, train_loss)], epochs=100, warmup_steps=100,show_progress_bar=True,
+          #output_path='完成2best-6-30',
+          checkpoint_path='checkpoint-savename',checkpoint_save_steps=6699,#どのくらいのイテレーションごとに保存するか
+          save_best_model=True#,#,#checkpoint_save_total_limit=5
+          #optimizer_params= {'lr': 2e-6}#
+
+          )
+model.save("savename")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#

ref.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a58f832b2017384ca7836311325d30f53554c3e6af82613bdfa9b9432d849be3
+size 13617545

src.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71dc30710bc43bcb09d545a40c51d591083e7b8c79b619b74e090919ddcbe4e7
+size 14587716

trg.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b082586b132398c03c6c84160466512a9a1f8de250ecdafa09ede38feaea1269
+size 12921129