from transformers import BertTokenizer, TFBertModel
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics import classification_report

# Load Quora dataset
quora_data = pd.read_csv('questions.csv', nrows=6000)

# Select a subset of the dataset for training
#train_data = quora_data.sample(n=80)  # Adjust the number of samples based on your requirements

# Get the minimum length between question1 and question2
min_length = min(len(quora_data['question1']), len(quora_data['question2']))

# Adjust the number of samples based on the minimum length
train_data = quora_data.sample(n=min_length)

# Convert the question pairs to list format
question1_list = list(train_data['question1'])
question2_list = list(train_data['question2'])
is_duplicate = list(train_data['is_duplicate'])

# Create DataFrame for the first pair
data1 = pd.DataFrame({'Sentences': question1_list, 'Labels': is_duplicate})

# Create DataFrame for the second pair
data2 = pd.DataFrame({'Sentences': question2_list, 'Labels': is_duplicate})

# Concatenate the two DataFrames
data = pd.concat([data1, data2], ignore_index=True)

# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(data['Sentences'], data['Labels'], test_size=0.2, random_state=42)

# Initialize the TF-IDF vectorizer
vectorizer = TfidfVectorizer()

# Vectorize the sentences in train set
X_train_vectors = vectorizer.fit_transform(X_train)

# Vectorize the sentences in test set
X_test_vectors = vectorizer.transform(X_test)

# Initialize the SVM classifier
svm = SVC()

# Fit the classifier to the training data
svm.fit(X_train_vectors, y_train)

# Predict labels for the test data
y_pred = svm.predict(X_test_vectors)

# Print classification report
print(classification_report(y_test, y_pred))

#svm_predictions = y_pred
# Save the ‘is_duplicate’ column in an Excel file
#result_df = pd.DataFrame({'is_duplicate': y_pred})
#result_df.to_excel('output.xlsx', index=False)

# Save the questions and classification in an Excel file
#data.to_excel(‘questions_classification.xlsx’, index=False)

# Create DataFrame for test predictions
result_df = pd.DataFrame({
    'Question1': list(test_data['question1']),
    'Question2': list(test_data['question2']),
    'is_duplicate': predictions.flatten()
})

# Create DataFrame for test predictions
#test_predictions = pd.DataFrame({'Question1': list(test_data['question1']), 'Question2': list(test_data['question2']), 'is_duplicate': predictions})

 # Create DataFrame for test predictions
#test_predictions = pd.DataFrame({'Question1': test_data['question1'], 'Question2': test_data['question2'], 'is_duplicate': predictions})

# Save the test predictions in an Excel file
result_df.to_excel('test_predictions_SVM.xlsx', index=False)
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, f1_score, recall_score

# Calculate metrics
accuracy = accuracy_score(y_test, y_pred)
cm = confusion_matrix(y_test, y_pred)
precision = precision_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)

# Print the results:
print("Accuracy:", accuracy)
print("Confusion Matrix:")
print(cm)
print("Precision:", precision)
print("F1 Score:", f1)
print("Recall:", recall)

NameError: name 'test_data' is not defined