Spaces:
Build error
Build error
from datasets import load_dataset | |
import pandas as pd | |
import torch | |
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler | |
from transformers import BertTokenizer, BertForQuestionAnswering, BertConfig,AutoModelForCausalLM | |
from pymongo import MongoClient | |
import torchtext | |
torchtext.disable_torchtext_deprecation_warning() | |
from torchtext.data import get_tokenizer | |
from yeni_tokenize import TokenizerProcessor | |
class Database: | |
# MongoDB connection settings | |
def get_mongodb(database_name='yeniDatabase', collection_name='test', host='localhost', port=27017): | |
""" | |
MongoDB connection and collection selection | |
""" | |
client = MongoClient(f'mongodb://{host}:{port}/') | |
db = client[database_name] | |
collection = db[collection_name] | |
return collection | |
def get_mongodb(): | |
# MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmalıdır. | |
return 'mongodb://localhost:27017/', 'yeniDatabase', 'train' | |
def get_input_texts(): | |
# MongoDB bağlantı bilgilerini alma | |
mongo_url, db_name, collection_name = Database.get_mongodb() | |
# MongoDB'ye bağlanma | |
client = MongoClient(mongo_url) | |
db = client[db_name] | |
collection = db[collection_name] | |
# Sorguyu tanımlama | |
query = {"Prompt": {"$exists": True}} | |
# Sorguyu çalıştırma ve dökümanları çekme | |
cursor = collection.find(query, {"Prompt": 1, "_id": 0}) | |
# Cursor'ı döküman listesine dönüştürme | |
input_texts_from_db = [doc['Prompt'] for doc in cursor] | |
# Input text'leri döndürme | |
# Düz metin listesine dönüştürme | |
return input_texts_from_db | |
input_text= get_input_texts() | |
print("metinler yazılıyor:") | |
for text in input_text: | |
print(text) | |
def get_output_texts(): | |
# MongoDB bağlantı bilgilerini alma | |
mongo_url, db_name, collection_name = Database.get_mongodb() | |
# MongoDB'ye bağlanma | |
client = MongoClient(mongo_url) | |
db = client[db_name] | |
collection = db[collection_name] | |
# Sorguyu tanımlama | |
query = {"Response": {"$exists": True}} | |
# Sorguyu çalıştırma ve dökümanları çekme | |
cursor = collection.find(query, {"Response": 1, "_id": 0}) | |
# Cursor'ı döküman listesine dönüştürme | |
output_texts_from_db = [doc['Response'] for doc in cursor] | |
#output metin listesine çevirme | |
return output_texts_from_db | |
def get_average_prompt_token_length(): | |
# MongoDB bağlantı bilgilerini alma | |
mongo_url, db_name, collection_name = Database.get_mongodb() | |
# MongoDB'ye bağlanma | |
client = MongoClient(mongo_url) | |
db = client[db_name] | |
collection = db[collection_name] | |
# Tüm dökümanları çekme ve 'prompt_token_length' alanını alma | |
docs = collection.find({}, {'Prompt_token_length': 1}) | |
# 'prompt_token_length' değerlerini toplama ve sayma | |
total_length = 0 | |
count = 0 | |
for doc in docs: | |
if 'Prompt_token_length' in doc: | |
total_length += doc['Prompt_token_length'] | |
count += 1 | |
# Ortalama hesaplama | |
average_length = total_length / count if count > 0 else 0 | |
return int(average_length) | |
# Tokenizer ve Modeli yükleme | |
""" | |
class TokenizerProcessor: | |
def __init__(self, tokenizer_name='bert-base-uncased'): | |
self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name) | |
def tokenize_and_encode(self, input_texts, output_texts, max_length=100): | |
encoded = self.tokenizer.batch_encode_plus( | |
text_pair=list(zip(input_texts, output_texts)), | |
padding='max_length', | |
truncation=True, | |
max_length=max_length, | |
return_attention_mask=True, | |
return_tensors='pt' | |
) | |
return encoded | |
paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="pt") | |
not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="pt") | |
paraphrase_classification_logits = model(**paraphrase)[0] | |
not_paraphrase_classification_logits = model(**not_paraphrase)[0] | |
def custom_padding(self, input_ids_list, max_length=100, pad_token_id=0): | |
padded_inputs = [] | |
for ids in input_ids_list: | |
if len(ids) < max_length: | |
padded_ids = ids + [pad_token_id] * (max_length - len(ids)) | |
else: | |
padded_ids = ids[:max_length] | |
padded_inputs.append(padded_ids) | |
return padded_inputs | |
def pad_and_truncate_pairs(self, input_texts, output_texts, max_length=100): | |
#input ve output verilerinin uzunluğunu eşitleme | |
inputs = self.tokenizer(input_texts, padding=False, truncation=False, return_tensors=None) | |
outputs = self.tokenizer(output_texts, padding=False, truncation=False, return_tensors=None) | |
input_ids = self.custom_padding(inputs['input_ids'], max_length, self.tokenizer.pad_token_id) | |
output_ids = self.custom_padding(outputs['input_ids'], max_length, self.tokenizer.pad_token_id) | |
input_ids_tensor = torch.tensor(input_ids) | |
output_ids_tensor = torch.tensor(output_ids) | |
input_attention_mask = (input_ids_tensor != self.tokenizer.pad_token_id).long() | |
output_attention_mask = (output_ids_tensor != self.tokenizer.pad_token_id).long() | |
return { | |
'input_ids': input_ids_tensor, | |
'input_attention_mask': input_attention_mask, | |
'output_ids': output_ids_tensor, | |
'output_attention_mask': output_attention_mask | |
} | |
""" | |
#cümleleri teker teker input ve output verilerinden çekmem gerekiyor | |
#def tokenize_and_pad_sequences(sequence_1,sequence2,) | |
"""class DataPipeline: | |
def __init__(self, tokenizer_name='bert-base-uncased', max_length=100): | |
self.tokenizer_processor = TokenizerProcessor(tokenizer_name) | |
self.max_length = max_length | |
def prepare_data(self): | |
input_texts = Database.get_input_texts() | |
output_texts = Database.get_output_texts() | |
encoded_data = self.tokenizer_processor.pad_and_truncate_pairs(input_texts, output_texts, self.max_length) | |
return encoded_data | |
def tokenize_texts(self, texts): | |
return [self.tokenize(text) for text in texts] | |
def encode_texts(self, texts): | |
return [self.encode(text, self.max_length) for text in texts] | |
# Example Usage | |
if __name__ == "__main__": | |
data_pipeline = DataPipeline() | |
encoded_data = data_pipeline.prepare_data() | |
print(encoded_data) | |
""" | |