konstantgr/themisto huggingface dataset:
import pandas as pd
features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])

train_features.fillna(0,inplace=True)
test_features.fillna(0,inplace=True)
validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
train_features
train_features

type(train_feature)
train_features

type(train_features)
!pip install sklearn
!pip install lightgbm as lbg
train_features.columns
target_drop = ["primary_label", "load_data", "helper_functions", 
          "data_preprocessing", "data_exploration", "modelling", 
          "prediction", "evaluation", "result_visualization", 
          "save_results", "comment_only"]

target = train_features["primary_label"]
train_features.drop(columns=target, inplace=True)
train_features.drop(columns=["filename"])
clf = lgb.LGBMClassifier()
!pip install sklearn
!pip install lightgbm as lgb
!pip install sklearn
!pip install lightgbm
clf.train(train[['cell_type', 'cell_number', 'execution_count',
       'linesofcomment', 'linesofcode', 'variable_count', 'function_count']])
import pandas as pd
import lightgbm as lgb
from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])

train_features.fillna(0, inplace=True)
test_features.fillna(0, inplace=True)
validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
clf.fit(train_features[['cell_type', 'cell_number', 'execution_count',
       'linesofcomment', 'linesofcode', 'variable_count', 'function_count']], target)
clf.fit(train_features[['cell_number', 'execution_count',
       'linesofcomment', 'linesofcode', 'variable_count', 'function_count']], target)
accuracy_score(clf.predict(train_features[['cell_number', 'execution_count',
       'linesofcomment', 'linesofcode', 'variable_count', 'function_count']]) target)
accuracy_score(clf.predict(train_features[['cell_number', 'execution_count',
       'linesofcomment', 'linesofcode', 'variable_count', 'function_count']]), target)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])

train_features.fillna(0, inplace=True)
test_features.fillna(0, inplace=True)
validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
train_columns = ['cell_number', 'execution_count', 'linesofcomment', 'linesofcode', 
                 'variable_count', 'function_count', 'display_data', 'stream', 'error']

clf.fit(train_features[train_columns], target)
accuracy_score(clf.predict(train_features[train_columns]), target)
target.value_counts()
target.value_counts(normalize=True)
accuracy_score(clf.predict(test_features[train_columns]), test_features["primary_label"])
train_features.columns
test_features.columns
train_features.columns
validation_features.columns
accuracy_score(clf.predict(validation_features[train_columns]), validation_features["primary_label"])
# accuracy_score(, validation_features["primary_label"])

pred = clf.predict(validation_features[train_columns])
from sklearn.metrics import f1_score

f1_score(pred, validation_features["primary_label"], average="weighted")
pred = clf.predict(validation_features[train_columns])
from sklearn.metrics import f1_score

f1_score(pred, validation_features["primary_label"], average="weighted")
pred = clf.predict(validation_features[train_columns])
from sklearn.metrics import f1_score

f1_score(pred, validation_features["primary_label"], average=None)
pred = clf.predict(validation_features[train_columns])
from sklearn.metrics import f1_score

# f1_score(pred, validation_features["primary_label"], average=None)

f1_score(pred, validation_features["primary_label"], average='weighted')
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
corpus = "I want some pitsaa"
X = vectorizer.fit_transform(corpus)
# X = vectorizer.fit_transform(corpus)
X
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
corpus = ["I want some pitsa"]
X = vectorizer.fit_transform(corpus)
# X = vectorizer.fit_transform(corpus)
X
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
corpus = ["I want some pitsa"]
X = vectorizer.fit_transform(corpus)
# X = vectorizer.fit_transform(corpus)
X.to_array()
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
corpus = ["I want some pitsa"]
X = vectorizer.fit_transform(corpus)
# X = vectorizer.fit_transform(corpus)
X.to_list()
train_features.text[1]
train_features.sample(20)
train_features.sample(20)[["code_line_before"]]
train_features.sample(30)[["code_line_before"]]
train_features.sample(30)[["text"]]
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = df["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

type(text)
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

str(text)
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text.apply(lambda x: " ".join(x))
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text.apply(lambda x: " ".join(list(x))
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["code_line_before"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text.apply(lambda x: " ".join(list(x)))
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features.fillna(0, inplace=True)
train_features[train_features["code_line_after"] == 0]["code_line_after"] = []

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features.fillna(0, inplace=True)
train_features[train_features["code_line_after"] == 0]

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features.fillna(0, inplace=True)
train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = []

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features.fillna(0, inplace=True)
train_features[train_features["code_line_after"] == 0]["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features.fillna(0, inplace=True)
train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
# train_features.drop(columns=["filename"])
train_features.columns
# train_features.drop(columns=["filename"])
train_features
# train_features.drop(columns=["filename"])
train_features["code_line_after"]
train_features["packages_info"]
train_features["packages_info"].unique()
train_features["packages_info"] != []
train_features["packages_info"] != "[]""
train_features["packages_info"] != "[]"
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment',
       'code_line_before', 'code_line_after', 'markdown_heading']].fillna([], inplace=True)
train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0,inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment',
       'code_line_before', 'code_line_after', 'markdown_heading']].fillna("", inplace=True)

train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment',
       'code_line_before', 'code_line_after', 'markdown_heading']] = train_features[['text', 'comment',
       'code_line_before', 'code_line_after', 'markdown_heading']].fillna("", inplace=True)

train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment', 'code_line_before', 
        'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
                                                                  'code_line_after', 'markdown_heading']].fillna("", inplace=True)

train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment', 'code_line_before', 
        'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
                                                                  'code_line_after', 'markdown_heading']].fillna("")

train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
train_features["code_line_after"]
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


train_features[['text', 'comment', 'code_line_before', 
        'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
                                                                  'code_line_after', 'markdown_heading']].fillna("NONE")

train_features.fillna(0, inplace=True)

# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
train_features["code_line_after"].sample(20)
(train_features["code_line_after"] == "NONE").sum()
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


# train_features[['text', 'comment', 'code_line_before', 
#         'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
#                                                                   'code_line_after', 'markdown_heading']].fillna("NONE")

train_features.fillna(0, inplace=True)
train_features[train_features['text'] == 0].loc["text"] = "NONE" 
# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


# train_features[['text', 'comment', 'code_line_before', 
#         'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
#                                                                   'code_line_after', 'markdown_heading']].fillna("NONE")

train_features.fillna(0, inplace=True)
train_features[train_features['text'] == 0]["text"] = "NONE" 
# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
(train_features["text"] == "NONE").sum()
import pandas as pd
import lightgbm as lgb
from sklearn.metrics import accuracy_score

from sklearn.model_selection import train_test_split

features_path = 'data/task2/'
test_features = pd.read_pickle(features_path+'test_features.pkl')
train_features = pd.read_pickle(features_path+'train_features.pkl')
validation_features = pd.read_pickle(features_path+'validation_features.pkl')

train_features.index = range(train_features.shape[0])
validation_features.index = range(validation_features.shape[0])
test_features.index = range(test_features.shape[0])


# train_features[['text', 'comment', 'code_line_before', 
#         'code_line_after', 'markdown_heading']] = train_features[['text', 'comment', 'code_line_before', 
#                                                                   'code_line_after', 'markdown_heading']].fillna("NONE")

train_features.fillna(0, inplace=True)
train_features[train_features['text'] == 0]['text'] = "NONE" 
# train_features[train_features["code_line_after"] == 0].loc["code_line_after"] = ""

test_features.fillna(0, inplace=True)

validation_features.fillna(0, inplace=True)

print(train_features.shape)
print(validation_features.shape)
print(test_features.shape)
(train_features["text"] == "NONE")
(train_features["text"])
(train_features["text"]).sample(20)
(train_features["text"]).sample(40)
(train_features["text"]).sample(100)
(train_features["text"]).sample(100)[0]
(train_features["text"]).sample(100).iloc[0]
type((train_features["text"]).sample(100).iloc[0])
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["text"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text.apply(lambda x: " ".join(x))
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

text = train_features["text"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

text.apply(lambda x: " ".join(x), inplace=True)
train_features.text[1]
from sklearn.feature_extraction.text import TfidfVectorizer

# text = train_features["text"]
# vectorizer = TfidfVectorizer()
# X = vectorizer.fit_transform(corpus)

train_features["text"] = text.apply(lambda x: " ".join(x))
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer()
vectorizer.fit_transform(train_features["text"])
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
import scipy

vectorizer = TfidfVectorizer()

X = scipy.sparse.csr_matrix(validation_features[train_columns].values)

vectorizer.fit_transform(train_features["text"])
# sklearn.linear_model.LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
import scipy
from scipy.sparse import hstack

vectorizer = TfidfVectorizer()

X = scipy.sparse.csr_matrix(validation_features[train_columns].values)

X2 = vectorizer.fit_transform(train_features["text"])

X3 = hstack((X, X2))