У меня есть модель для LLM, которая использует TF-IDF + LogReg. У меня есть оригинальная версия и модифицированная. Покажи мне все отличия между версиями, а также их предназначение:
Оригинал:
!pip install datasets

!python -m spacy download ru_core_news_sm

from typing import List, Tuple

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
import spacy
from datasets import load_dataset

def load_sib200_ru() -> Tuple[Tuple[List[str], List[int]], Tuple[List[str], List[int]], Tuple[List[str], List[int]], List[str]]:
trainset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='train')
X_train = trainset['text']
y_train = trainset['category']
valset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='validation')
X_val = valset['text']
y_val = valset['category']
testset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='test')
X_test = testset['text']
y_test = testset['category']
categories = set(y_train)
unknown_categories = set(y_val) - categories
if len(unknown_categories) > 0:
err_msg = f'The categories {unknown_categories} are represented in the validation set, but they are not represented in the training set.'
raise RuntimeError(err_msg)
unknown_categories = set(y_test) - categories
if len(unknown_categories) > 0:
err_msg = f'The categories {unknown_categories} are represented in the test set, but they are not represented in the training set.'
raise RuntimeError(err_msg)
categories = sorted(list(categories))
y_train = [categories.index(it) for it in y_train]
y_val = [categories.index(it) for it in y_val]
y_test = [categories.index(it) for it in y_test]
return (X_train, y_train), (X_val, y_val), (X_test, y_test), categories

def normalize_text(s: str, nlp_pipeline: spacy.Language) -> str:
doc = nlp_pipeline(s)
lemmas = [('' if token.like_num else token.lemma_.lower()) for token in filter(lambda it1: not it1.is_punct, doc)]
if len(lemmas) == 0:
return ''
return ' '.join(lemmas)

train_data, val_data, test_data, classes_list = load_sib200_ru()

print(f'Categories: {classes_list}')

print(len(train_data[0]))
print(len(train_data[1]))

print(len(val_data[0]))
print(len(val_data[1]))

print(len(test_data[0]))
print(len(test_data[1]))

nlp = spacy.load('ru_core_news_sm')

print(train_data[0][0])

print(normalize_text(train_data[0][0], nlp))

print(val_data[0][0])

print(normalize_text(val_data[0][0], nlp))

print(test_data[0][0])

print(normalize_text(test_data[0][0], nlp))

class_probability = 1.0 / len(classes_list)
max_df = 1.0 - 0.2 * class_probability
print(f'Maximal document frequency of term is {max_df}.')

classifier = Pipeline(steps=[
('vectorizer', TfidfVectorizer(token_pattern='\w+', max_df=max_df, min_df=1)),
('cls', LogisticRegression(solver='saga', max_iter=100, random_state=42))
])

cv = GridSearchCV(
estimator=classifier,
param_grid={
'vectorizer__ngram_range': [(1, 1), (1, 2), (1, 3)],
'cls__C': [1e-1, 1, 10, 100, 1000],
'cls__penalty': ['l1', 'l2']
},
scoring='f1_macro',
cv=5,
refit=True,
n_jobs=-1,
verbose=True
)

cv.fit([normalize_text(it, nlp) for it in train_data[0]], train_data[1])

print('Best parameters:')
print(cv.best_params_)

print('Best F1-macro:')
print(cv.best_score_)

print(f'Vocabulary size is {len(cv.best_estimator_.named_steps["vectorizer"].vocabulary_)}.')

y_pred = cv.predict([normalize_text(it, nlp) for it in val_data[0]])
print(classification_report(y_true=val_data[1], y_pred=y_pred, target_names=classes_list))

y_pred = cv.predict([normalize_text(it, nlp) for it in test_data[0]])
print(classification_report(y_true=test_data[1], y_pred=y_pred, target_names=classes_list))

Модифицированная:
from typing import List, Tuple
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
import spacy
from datasets import load_dataset

!pip install -U spacy
!python -m spacy download ru_core_news_sm # pre-trained lang model

def load_sib200_ru() -> Tuple[Tuple[List[str], List[int]], Tuple[List[str], List[int]], Tuple[List[str], List[int]], List[str]]:
'''
loads sib200_ru dataset
Returns three tuples (for training, validation and testing) and a list of all categories
'''


# Loading data: training set of text and corresponding categories from the dataset
trainset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='train')
X_train = trainset['text']
y_train = trainset['category']

# Loading data: validation set of text and corresponding categories from the dataset
valset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='validation')
X_val = valset['text']
y_val = valset['category']

# Loading data: test set of text and corresponding categories from the dataset
testset = load_dataset('Davlan/sib200', 'rus_Cyrl', split='test')
X_test = testset['text']
y_test = testset['category']

# Checks: if categs from val ant test are in train
categories = set(y_train)
unknown_categories = set(y_val) - categories
if len(unknown_categories) > 0:
    raise RuntimeError(f'The categories {unknown_categories} are represented in the validation set, but they are not represented in the training set.')
unknown_categories = set(y_test) - categories
if len(unknown_categories) > 0:
    raise RuntimeError(f'The categories {unknown_categories} are represented in the test set, but they are not represented in the training set.')

# categ -> idxs ("sport", "politics", "economy" -> 0, 1, 2)
categories = sorted(list(categories))
y_train = [categories.index(it) for it in y_train]
y_val = [categories.index(it) for it in y_val]
y_test = [categories.index(it) for it in y_test]

return (X_train, y_train), (X_val, y_val), (X_test, y_test), categories
New: removing stop words
def normalize_text(s: str, nlp_pipeline: spacy.Language) -> str:
'''
normalizes text using nlp_pipeline
'''


# Process each token in the document
doc = nlp_pipeline(s)
lemmas = [
    ('<NUM>' if token.like_num else token.lemma_.lower())
    for token in filter(lambda token: not token.is_punct and not token.is_stop, doc)
]

if len(lemmas) == 0:
    return ''
return ' '.join(lemmas)
load data set
train_data, val_data, test_data, classes_list = load_sib200_ru()
nlp = spacy.load('ru_core_news_sm')

print(f'Categories: {classes_list}')

print(len(train_data[0]))
print(len(train_data[1]))

print(len(val_data[0]))
print(len(val_data[1]))

print(len(test_data[0]))
print(len(test_data[1]))

print(train_data[0][0])

print(normalize_text(train_data[0][0], nlp))

print(val_data[0][0])

print(normalize_text(val_data[0][0], nlp))

print(test_data[0][0])

print(normalize_text(test_data[0][0], nlp))

Find maximum term frequency in documents
class_probability = 1.0 / len(classes_list)
max_df = 1.0 - 0.2 * class_probability
print(f'Maximal document frequency of term is {max_df}.')

Classifier:
#1) Text vectorization using TfidfVectorizer, which converts text data into a numeric format taking into account TF-IDF
#2) Train logistic regression as a classification model
classifier = Pipeline(steps=[
('vectorizer', TfidfVectorizer(token_pattern='\w+', max_df=max_df, min_df=1)),
('cls', LogisticRegression(solver='saga', max_iter=1000, random_state=42)) # New: Increased number of iterations (100 -> 1000)
])

cv = GridSearchCV(
estimator=classifier,
param_grid={
'vectorizer__ngram_range': [(1, 1), (1, 2), (1, 3)], # ngram range (uni-/bi-/tri-)
'vectorizer__sublinear_tf': [True, False], # New: TF (term frequency)
'cls__C': [0.01, 0.1, 1, 10, 100, 1000], # regularization parameter (New: 0.01)
'cls__penalty': ['l1', 'l2'] # type of regul (lasso/ridge)
},
scoring='f1_macro',
cv=5,
refit=True,
n_jobs=-1,
verbose=True
)

Train model on the normalized texts
cv.fit([normalize_text(it, nlp) for it in train_data[0]], train_data[1])

print('Best parameters:')
print(cv.best_params_)
print('Best F1-macro:')
print(cv.best_score_)
print(f'Vocabulary size is {len(cv.best_estimator_.named_steps["vectorizer"].vocabulary_)}.')

y_pred = cv.predict([normalize_text(it, nlp) for it in val_data[0]])
print(classification_report(y_true=val_data[1], y_pred=y_pred, target_names=classes_list))

y_pred = cv.predict([normalize_text(it, nlp) for it in test_data[0]])
print(classification_report(y_true=test_data[1], y_pred=y_pred, target_names=classes_list))