Spaces:
Sleeping
Sleeping
Add L1 regularization
Browse files- chatbot_constructor.py +13 -12
chatbot_constructor.py
CHANGED
@@ -29,7 +29,7 @@ def todset(text: str):
|
|
29 |
def hash_str(data: str):
|
30 |
return hashlib.md5(data.encode('utf-8')).hexdigest()
|
31 |
|
32 |
-
def train(message: str = "", dropout: float = 0.
|
33 |
data_hash = None
|
34 |
if "→" not in data or "\n" not in data:
|
35 |
if data in os.listdir("cache"):
|
@@ -44,7 +44,7 @@ def train(message: str = "", dropout: float = 0.5, learning_rate: float = 0.001,
|
|
44 |
vocab_size = len(tokenizer.word_index) + 1
|
45 |
inp_len = input_len
|
46 |
if data_hash is None:
|
47 |
-
data_hash = hash_str(data)+"_"+str(dropout)+"_"+str(learning_rate)+"_"+str(epochs)+"_"+str(emb_size)+"_"+str(inp_len)+"_"+str(kernels_count)+"_"+str(kernel_size)+".keras"
|
48 |
elif message == "!getmodelhash":
|
49 |
return data_hash
|
50 |
else:
|
@@ -57,9 +57,9 @@ def train(message: str = "", dropout: float = 0.5, learning_rate: float = 0.001,
|
|
57 |
dropout1_layer = Dropout(dropout)(emb_layer)
|
58 |
attn_layer = MultiHeadAttention(num_heads=4, key_dim=128)(dropout1_layer, dropout1_layer, dropout1_layer)
|
59 |
noise_layer = GaussianNoise(0.1)(attn_layer)
|
60 |
-
conv1_layer = Conv1D(kernels_count, kernel_size, padding='same', activation='relu', strides=1, input_shape=(64, 128))(noise_layer)
|
61 |
-
conv2_layer = Conv1D(16, 4, padding='same', activation='relu', strides=1)(conv1_layer)
|
62 |
-
conv3_layer = Conv1D(8, 2, padding='same', activation='relu', strides=1)(conv2_layer)
|
63 |
flatten_layer = Flatten()(conv3_layer)
|
64 |
attn_flatten_layer = Flatten()(attn_layer)
|
65 |
conv1_flatten_layer = Flatten()(conv1_layer)
|
@@ -67,16 +67,16 @@ def train(message: str = "", dropout: float = 0.5, learning_rate: float = 0.001,
|
|
67 |
conv3_flatten_layer = Flatten()(conv3_layer)
|
68 |
concat1_layer = Concatenate()([flatten_layer, attn_flatten_layer, conv1_flatten_layer, conv2_flatten_layer, conv3_flatten_layer])
|
69 |
dropout2_layer = Dropout(dropout)(concat1_layer)
|
70 |
-
dense1_layer = Dense(512, activation="linear")(dropout2_layer)
|
71 |
prelu1_layer = PReLU()(dense1_layer)
|
72 |
dropout3_layer = Dropout(dropout)(prelu1_layer)
|
73 |
-
dense2_layer = Dense(256, activation="tanh")(dropout3_layer)
|
74 |
dropout4_layer = Dropout(dropout)(dense2_layer)
|
75 |
-
dense3_layer = Dense(256, activation="relu")(dropout4_layer)
|
76 |
dropout5_layer = Dropout(dropout)(dense3_layer)
|
77 |
-
dense4_layer = Dense(100, activation="tanh")(dropout5_layer)
|
78 |
concat2_layer = Concatenate()([dense4_layer, prelu1_layer, attn_flatten_layer, conv1_flatten_layer])
|
79 |
-
dense4_layer = Dense(resps_len, activation="softmax")(concat2_layer)
|
80 |
model = Model(inputs=input_layer, outputs=dense4_layer)
|
81 |
|
82 |
X = []
|
@@ -101,8 +101,9 @@ def train(message: str = "", dropout: float = 0.5, learning_rate: float = 0.001,
|
|
101 |
|
102 |
if __name__ == "__main__":
|
103 |
iface = gr.Interface(fn=train, inputs=["text",
|
104 |
-
gr.inputs.Slider(0, 0.
|
105 |
-
gr.inputs.Slider(0
|
|
|
106 |
gr.inputs.Slider(1, 64, default=32, step=1, label="Epochs"),
|
107 |
gr.inputs.Slider(1, 256, default=100, step=1, label="Embedding size"),
|
108 |
gr.inputs.Slider(1, 128, default=16, step=1, label="Input Length"),
|
|
|
29 |
def hash_str(data: str):
|
30 |
return hashlib.md5(data.encode('utf-8')).hexdigest()
|
31 |
|
32 |
+
def train(message: str = "", regularization: float = 0.0001, dropout: float = 0.1, learning_rate: float = 0.001, epochs: int = 16, emb_size: int = 128, input_len: int = 16, kernels_count: int = 8, kernel_size: int = 8, data: str = ""):
|
33 |
data_hash = None
|
34 |
if "→" not in data or "\n" not in data:
|
35 |
if data in os.listdir("cache"):
|
|
|
44 |
vocab_size = len(tokenizer.word_index) + 1
|
45 |
inp_len = input_len
|
46 |
if data_hash is None:
|
47 |
+
data_hash = hash_str(data)+"_"+str(l1)+"_"+str(dropout)+"_"+str(learning_rate)+"_"+str(epochs)+"_"+str(emb_size)+"_"+str(inp_len)+"_"+str(kernels_count)+"_"+str(kernel_size)+".keras"
|
48 |
elif message == "!getmodelhash":
|
49 |
return data_hash
|
50 |
else:
|
|
|
57 |
dropout1_layer = Dropout(dropout)(emb_layer)
|
58 |
attn_layer = MultiHeadAttention(num_heads=4, key_dim=128)(dropout1_layer, dropout1_layer, dropout1_layer)
|
59 |
noise_layer = GaussianNoise(0.1)(attn_layer)
|
60 |
+
conv1_layer = Conv1D(kernels_count, kernel_size, padding='same', activation='relu', strides=1, input_shape=(64, 128), kernel_regularizer=regularizers.L1(regularization))(noise_layer)
|
61 |
+
conv2_layer = Conv1D(16, 4, padding='same', activation='relu', strides=1, kernel_regularizer=regularizers.L1(regularization))(conv1_layer)
|
62 |
+
conv3_layer = Conv1D(8, 2, padding='same', activation='relu', strides=1, kernel_regularizer=regularizers.L1(regularization))(conv2_layer)
|
63 |
flatten_layer = Flatten()(conv3_layer)
|
64 |
attn_flatten_layer = Flatten()(attn_layer)
|
65 |
conv1_flatten_layer = Flatten()(conv1_layer)
|
|
|
67 |
conv3_flatten_layer = Flatten()(conv3_layer)
|
68 |
concat1_layer = Concatenate()([flatten_layer, attn_flatten_layer, conv1_flatten_layer, conv2_flatten_layer, conv3_flatten_layer])
|
69 |
dropout2_layer = Dropout(dropout)(concat1_layer)
|
70 |
+
dense1_layer = Dense(512, activation="linear", kernel_regularizer=regularizers.L1(regularization))(dropout2_layer)
|
71 |
prelu1_layer = PReLU()(dense1_layer)
|
72 |
dropout3_layer = Dropout(dropout)(prelu1_layer)
|
73 |
+
dense2_layer = Dense(256, activation="tanh", kernel_regularizer=regularizers.L1(regularization))(dropout3_layer)
|
74 |
dropout4_layer = Dropout(dropout)(dense2_layer)
|
75 |
+
dense3_layer = Dense(256, activation="relu", kernel_regularizer=regularizers.L1(regularization))(dropout4_layer)
|
76 |
dropout5_layer = Dropout(dropout)(dense3_layer)
|
77 |
+
dense4_layer = Dense(100, activation="tanh", kernel_regularizer=regularizers.L1(regularization))(dropout5_layer)
|
78 |
concat2_layer = Concatenate()([dense4_layer, prelu1_layer, attn_flatten_layer, conv1_flatten_layer])
|
79 |
+
dense4_layer = Dense(resps_len, activation="softmax", kernel_regularizer=regularizers.L1(regularization))(concat2_layer)
|
80 |
model = Model(inputs=input_layer, outputs=dense4_layer)
|
81 |
|
82 |
X = []
|
|
|
101 |
|
102 |
if __name__ == "__main__":
|
103 |
iface = gr.Interface(fn=train, inputs=["text",
|
104 |
+
gr.inputs.Slider(0, 0.01, default=0.0001, step=0.00000001, label="Regularization L1"),
|
105 |
+
gr.inputs.Slider(0, 0.5, default=0.1, step=0.00000001, label="Dropout"),
|
106 |
+
gr.inputs.Slider(0.00000001, 0.01, default=0.001, step=0.00000001, label="Learning rate"),
|
107 |
gr.inputs.Slider(1, 64, default=32, step=1, label="Epochs"),
|
108 |
gr.inputs.Slider(1, 256, default=100, step=1, label="Embedding size"),
|
109 |
gr.inputs.Slider(1, 128, default=16, step=1, label="Input Length"),
|