Upload 9 files

data_preparing.py

@@ -0,0 +1,26 @@
+from sklearn.feature_extraction.text import CountVectorizer
+from data_analysis import df
+from sklearn.preprocessing import LabelEncoder
+from data_splitting import y_train, y_val
+import tensorflow as tf
+
+#Input Variable
+# vectorizing input varible 'clean_text' into a matrix 
+features = df['clean_text']
+
+cv = CountVectorizer() # ngram_range=(1,2)
+features = cv.fit_transform(features)
+
+# changing the datatype of the number into uint8 to consume less memory
+features = features.astype('uint8') # uint8 and float32
+
+
+# defining target variable
+# using LabelEncoder to get placeholder number values for categorical variabel 'language'
+le = LabelEncoder()
+df['language_encoded'] = le.fit_transform(df['language'])
+
+targets = df['language_encoded']
+
+y_train_encoded = tf.keras.utils.to_categorical(y_train, num_classes=22)
+y_val_encoded = tf.keras.utils.to_categorical(y_val, num_classes=22)

data_splitting.py

@@ -0,0 +1,12 @@
+from sklearn.model_selection import train_test_split
+from data_preparing import features, targets
+from data_preparing import le
+# splitting data into training and testing datasets
+
+X_train, X_val, y_train, y_val = train_test_split(features, targets, test_size=0.2, random_state=2007)
+
+X_train = X_train.toarray()
+X_val = X_val.toarray()
+
+input_size = X_train.shape[1]
+num_classes = len(le.classes_)

model.py

@@ -0,0 +1,13 @@
+import tensorflow as tf
+from data_splitting import num_classes, input_size
+
+
+model = tf.keras.models.Sequential([
+    tf.keras.layers.Dense(100, activation='relu', kernel_initializer='he_normal', kernel_regularizer=tf.keras.regularizers.l2(0.01), input_shape=(input_size,)),
+    tf.keras.layers.BatchNormalization(),
+    tf.keras.layers.Dense(80, activation='relu', kernel_initializer='he_normal', kernel_regularizer=tf.keras.regularizers.l2(0.01)),
+    tf.keras.layers.BatchNormalization(),
+    tf.keras.layers.Dense(50, activation='relu', kernel_initializer='he_normal', kernel_regularizer=tf.keras.regularizers.l2(0.01)),
+    tf.keras.layers.BatchNormalization(),
+    tf.keras.layers.Dense(num_classes, activation='softmax')
+])

training.py

@@ -0,0 +1,10 @@
+from model import model
+from data_splitting import num_classes, X_train, X_val, input_size
+from data_preparing import y_train_encoded, y_val_encoded
+from model_callbacks import optimizer, early_stopping, lr_scheduler_callback
+
+model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
+
+model.fit(X_train, y_train_encoded, epochs=10, batch_size=32, validation_data=(X_val, y_val_encoded), callbacks=[early_stopping,lr_scheduler_callback])
+
+model.save('full_language_identifcation_model1.h5')