Spaces:
Runtime error
Runtime error
File size: 5,114 Bytes
b27edec 4e01b20 b27edec 22ef6fa b27edec 231b9bc b27edec 22ef6fa b27edec 22ef6fa b27edec ce81925 b27edec ce81925 a3533fc b27edec 22ef6fa b27edec |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
from transformers import AutoTokenizer, AutoModelForSequenceClassification, DataCollatorWithPadding
from datasets import load_dataset, load_metric
import evaluate
from torch.utils.data import DataLoader
import torch
import numpy as np
import pandas as pd
import options as op
def tp_tf_test(metric_selector, test_dataset, model_selector, queries_selector, prompt_selector, prediction_strategy_selector):
#Load test dataset___________________________
test_dataset = load_dataset(test_dataset)['test']
#Load queries________________________________
queries_data_file = queries_selector.split('-')[-1]
queries_dataset_path = queries_selector.replace('-'+queries_data_file, '')
queries_dataset_split = {'queries': queries_data_file}
queries_dataset = load_dataset(queries_dataset_path, data_files = queries_dataset_split)['queries']
#Load prompt_________________________________
prompt = prompt_selector
#Load prediction strategias__________________
prediction_strategies = prediction_strategy_selector
#Load model, tokenizer and collator__________
model = AutoModelForSequenceClassification.from_pretrained(model_selector)
if torch.cuda.is_available():
device = torch.device("cuda")
model.to(device)
tokenizer = AutoTokenizer.from_pretrained(model_selector)
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
#Calculate and save predictions______________
#'''
def tokenize_function(example, prompt = '', query = ''):
queries = []
for i in range(len(example['title'])):
queries.append(prompt + query)
tokenize = tokenizer(example['title'], queries, truncation='only_first')
#tokenize['query'] = queries
return tokenize
results_test = pd.DataFrame()
for query_data in queries_dataset:
query = query_data['query']
tokenized_test_dataset = test_dataset.map(tokenize_function, batched = True, fn_kwargs = {'prompt' : prompt, 'query' : query})
columns_to_remove = test_dataset.column_names
for column_name in ['label_ids', 'nli_label']:
columns_to_remove.remove(column_name)
tokenized_test_dataset_for_inference = tokenized_test_dataset.remove_columns(columns_to_remove)
tokenized_test_dataset_for_inference.set_format('torch')
dataloader = DataLoader(
tokenized_test_dataset_for_inference,
batch_size=8,
collate_fn = data_collator,
)
values = []
labels = []
nli_labels =[]
for batch in dataloader:
if torch.cuda.is_available():
data = {k: v.to(device) for k, v in batch.items() if k not in ['labels', 'nli_label']}
else:
data = {k: v for k, v in batch.items() if k not in ['labels', 'nli_label']}
with torch.no_grad():
outputs = model(**data)
logits = outputs.logits
entail_contradiction_logits = logits[:,[0,2]]
probs = entail_contradiction_logits.softmax(dim=1)
predictions = probs[:,1].tolist()
label_ids = batch['labels'].tolist()
nli_label_ids = batch['nli_label'].tolist()
for prediction, label, nli_label in zip(predictions, label_ids, nli_label_ids):
values.append(prediction)
labels.append(label)
nli_labels.append(nli_label)
results_test['dataset_labels'] = labels
results_test['nli_labels'] = nli_labels
results_test[query] = values
results_test.to_csv('Reports/ZS inference tables/ZS-inference-table_Model-' + op.models[model_selector] + '_Queries-' + op.queries[queries_dataset_path][queries_data_file] + '_Prompt-' + op.prompts[prompt_selector] + '.csv', index = False)
#'''
#Load saved predictions____________________________
'''
results_test = pd.read_csv('Reports/ZS inference tables/ZS-inference-table_Model-' + op.models[model_selector] + '_Queries-' + op.queries[queries_dataset_path][queries_data_file] + '_Prompt-' + op.prompts[prompt_selector] + '.csv')
'''
#Analize predictions_______________________________
def logits_labels(raw):
classes = raw["dataset_labels"].unique()
raw_logits = raw.iloc[:,2:]
logits = np.zeros(shape=(len(raw_logits.index),len(classes)))
for i in range(len(classes)):
queries = queries_dataset.filter(lambda x: x['label_ids'] == i)['query']
logits[:,i]=raw_logits[queries].max(axis=1)
labels = raw[["dataset_labels","nli_labels"]]
labels = np.array(labels).astype(int)
return logits, labels
predictions, references = logits_labels(results_test)
prediction_strategies = [op.prediction_strategy_options[x] for x in prediction_strategy_selector]
metric = evaluate.load(metric_selector)
metric.add_batch(predictions = predictions, references = references)
results = metric.compute(prediction_strategies = prediction_strategies)
prediction_strategies_names = '-'.join(prediction_strategy_selector).replace(" ", "")
output_filename = 'Reports/report-Model-' + op.models[model_selector] + '_Queries-' + op.queries[queries_dataset_path][queries_data_file] + '_Prompt-' + op.prompts[prompt_selector] + '_Strategies-'+ prediction_strategies_names +'.csv'
with open(output_filename, 'a') as results_file:
for result in results:
results[result].to_csv(results_file, mode='a', index_label = result)
print(results[result], '\n')
return output_filename
|