taneemishere commited on
Commit
f6a67e3
1 Parent(s): 60010cc

added essential files and directories

Browse files
Files changed (47) hide show
  1. .DS_Store +0 -0
  2. app.py +43 -0
  3. bin/.DS_Store +0 -0
  4. bin/autoencoder.h5 +3 -0
  5. bin/autoencoder.json +1 -0
  6. bin/pix2code2.json +1 -0
  7. bin/words.vocab +20 -0
  8. classes/.DS_Store +0 -0
  9. classes/Sampler.py +116 -0
  10. classes/Utils.py +37 -0
  11. classes/Vocabulary.py +78 -0
  12. classes/__init__.py +0 -0
  13. classes/__pycache__/BeamSearch.cpython-35.pyc +0 -0
  14. classes/__pycache__/BeamSearch.cpython-38.pyc +0 -0
  15. classes/__pycache__/BeamSearch.cpython-39.pyc +0 -0
  16. classes/__pycache__/Sampler.cpython-35.pyc +0 -0
  17. classes/__pycache__/Sampler.cpython-38.pyc +0 -0
  18. classes/__pycache__/Sampler.cpython-39.pyc +0 -0
  19. classes/__pycache__/Utils.cpython-35.pyc +0 -0
  20. classes/__pycache__/Utils.cpython-38.pyc +0 -0
  21. classes/__pycache__/Utils.cpython-39.pyc +0 -0
  22. classes/__pycache__/Vocabulary.cpython-35.pyc +0 -0
  23. classes/__pycache__/Vocabulary.cpython-38.pyc +0 -0
  24. classes/__pycache__/Vocabulary.cpython-39.pyc +0 -0
  25. classes/__pycache__/__init__.cpython-35.pyc +0 -0
  26. classes/__pycache__/__init__.cpython-38.pyc +0 -0
  27. classes/__pycache__/__init__.cpython-39.pyc +0 -0
  28. classes/model/AModel.py +25 -0
  29. classes/model/Config.py +7 -0
  30. classes/model/__init__.py +0 -0
  31. classes/model/__pycache__/AModel.cpython-35.pyc +0 -0
  32. classes/model/__pycache__/AModel.cpython-38.pyc +0 -0
  33. classes/model/__pycache__/Config.cpython-35.pyc +0 -0
  34. classes/model/__pycache__/Config.cpython-38.pyc +0 -0
  35. classes/model/__pycache__/__init__.cpython-35.pyc +0 -0
  36. classes/model/__pycache__/__init__.cpython-38.pyc +0 -0
  37. classes/model/__pycache__/__init__.cpython-39.pyc +0 -0
  38. classes/model/__pycache__/autoencoder_image.cpython-35.pyc +0 -0
  39. classes/model/__pycache__/autoencoder_image.cpython-38.pyc +0 -0
  40. classes/model/__pycache__/pix2code.cpython-35.pyc +0 -0
  41. classes/model/__pycache__/pix2code2.cpython-35.pyc +0 -0
  42. classes/model/__pycache__/pix2code2.cpython-38.pyc +0 -0
  43. classes/model/__pycache__/pix2code2.cpython-39.pyc +0 -0
  44. classes/model/autoencoder_image.py +61 -0
  45. classes/model/pix2code2.py +67 -0
  46. data/.DS_Store +0 -0
  47. main_program.py +101 -0
.DS_Store ADDED
Binary file (8.2 kB). View file
 
app.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import main_program
3
+
4
+
5
+ # our model's i/o method that take image from gradio interface's inputs.Image()
6
+ def model_interface(image):
7
+ return main_model(image)
8
+
9
+
10
+ # main method that call the main_program where code is generated and then compiled
11
+ def main_model(input_image):
12
+ result = main_program.main_program(input_image)
13
+ return result
14
+
15
+
16
+ interface_title = "<br> <p style='margin: 0% 8% 0% 8%'>HTML Code Generation from Images with Deep Neural Networks</p>"
17
+ interface_description = """<p style='margin: 0% 8% 2% 8%; text-align: justify;text-justify: inter-word;'> Writing
18
+ code in a programming language for a designed mockup or a graphical user interface created by designers and UI
19
+ engineers, is done mostly by developers to build and develop custom websites and software. The development work is
20
+ not approachable by those unfamiliar with programming, to drive these personas capable of designing and developing
21
+ the code bases and website structures we come up with an automated system. In this work, we showed and proposed that
22
+ methods of deep learning and computer vision can be grasped to train a model that will automatically generate HTML
23
+ code from a single input mockup image and try to build an end-to-end automated system with around 85% of accuracy for
24
+ developing the structures of a web pages.</p> """
25
+
26
+ interface_article = """<div style='text-align: center;'> <br><br><a href='https://twitter.com/taneemishere'
27
+ target='_blank'>Developed by Taneem Jan</a> </div>
28
+ <div style='text-align: center;'> <a href='https://taneemishere.github.io'
29
+ target='_blank'>Paper</a> &ensp; &emsp; <a href='https://github.com/taneemishere'
30
+ target='_blank'>Code</a> </div>
31
+ """
32
+
33
+ # a gradio interface to convert a image to HTML Code
34
+ interface = gr.Interface(
35
+ model_interface,
36
+ inputs=gr.inputs.Image(),
37
+ outputs=gr.outputs.Textbox(),
38
+ title=interface_title,
39
+ description=interface_description,
40
+ article=interface_article
41
+ )
42
+
43
+ interface.launch(share=False)
bin/.DS_Store ADDED
Binary file (6.15 kB). View file
 
bin/autoencoder.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86d9ae3ae4c294424d3bf56f448e916893eb5374907a211d09712ec3476855b5
3
+ size 2901584
bin/autoencoder.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"keras_version": "2.1.2", "config": {"layers": [{"name": "input_1", "config": {"name": "input_1", "sparse": false, "batch_input_shape": [null, 256, 256, 3], "dtype": "float32"}, "class_name": "InputLayer", "inbound_nodes": []}, {"name": "conv2d_1", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 32, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_1", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"name": "conv2d_2", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 32, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_2", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["conv2d_1", 0, 0, {}]]]}, {"name": "max_pooling2d_1", "config": {"name": "max_pooling2d_1", "strides": [2, 2], "padding": "valid", "trainable": true, "pool_size": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D", "inbound_nodes": [[["conv2d_2", 0, 0, {}]]]}, {"name": "dropout_1", "config": {"name": "dropout_1", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["max_pooling2d_1", 0, 0, {}]]]}, {"name": "conv2d_3", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 64, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_3", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["dropout_1", 0, 0, {}]]]}, {"name": "conv2d_4", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 64, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_4", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["conv2d_3", 0, 0, {}]]]}, {"name": "max_pooling2d_2", "config": {"name": "max_pooling2d_2", "strides": [2, 2], "padding": "valid", "trainable": true, "pool_size": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D", "inbound_nodes": [[["conv2d_4", 0, 0, {}]]]}, {"name": "dropout_2", "config": {"name": "dropout_2", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["max_pooling2d_2", 0, 0, {}]]]}, {"name": "conv2d_5", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 128, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_5", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["dropout_2", 0, 0, {}]]]}, {"name": "conv2d_6", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 128, "kernel_regularizer": null, "kernel_size": [3, 3], "name": "conv2d_6", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "dilation_rate": [1, 1], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2D", "inbound_nodes": [[["conv2d_5", 0, 0, {}]]]}, {"name": "max_pooling2d_3", "config": {"name": "max_pooling2d_3", "strides": [2, 2], "padding": "valid", "trainable": true, "pool_size": [2, 2], "data_format": "channels_last"}, "class_name": "MaxPooling2D", "inbound_nodes": [[["conv2d_6", 0, 0, {}]]]}, {"name": "encoded_layer", "config": {"name": "encoded_layer", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["max_pooling2d_3", 0, 0, {}]]]}, {"name": "conv2d_transpose_1", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 128, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_1", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["encoded_layer", 0, 0, {}]]]}, {"name": "conv2d_transpose_2", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 128, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_2", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["conv2d_transpose_1", 0, 0, {}]]]}, {"name": "up_sampling2d_1", "config": {"size": [2, 2], "name": "up_sampling2d_1", "trainable": true, "data_format": "channels_last"}, "class_name": "UpSampling2D", "inbound_nodes": [[["conv2d_transpose_2", 0, 0, {}]]]}, {"name": "dropout_3", "config": {"name": "dropout_3", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["up_sampling2d_1", 0, 0, {}]]]}, {"name": "conv2d_transpose_3", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 64, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_3", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["dropout_3", 0, 0, {}]]]}, {"name": "conv2d_transpose_4", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 64, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_4", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["conv2d_transpose_3", 0, 0, {}]]]}, {"name": "up_sampling2d_2", "config": {"size": [2, 2], "name": "up_sampling2d_2", "trainable": true, "data_format": "channels_last"}, "class_name": "UpSampling2D", "inbound_nodes": [[["conv2d_transpose_4", 0, 0, {}]]]}, {"name": "dropout_4", "config": {"name": "dropout_4", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["up_sampling2d_2", 0, 0, {}]]]}, {"name": "conv2d_transpose_5", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 32, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_5", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["dropout_4", 0, 0, {}]]]}, {"name": "conv2d_transpose_6", "config": {"kernel_initializer": {"config": {"scale": 1.0, "seed": null, "mode": "fan_avg", "distribution": "uniform"}, "class_name": "VarianceScaling"}, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "use_bias": true, "filters": 3, "kernel_regularizer": null, "trainable": true, "name": "conv2d_transpose_6", "activity_regularizer": null, "strides": [1, 1], "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "padding": "same", "kernel_size": [3, 3], "activation": "relu", "data_format": "channels_last"}, "class_name": "Conv2DTranspose", "inbound_nodes": [[["conv2d_transpose_5", 0, 0, {}]]]}, {"name": "up_sampling2d_3", "config": {"size": [2, 2], "name": "up_sampling2d_3", "trainable": true, "data_format": "channels_last"}, "class_name": "UpSampling2D", "inbound_nodes": [[["conv2d_transpose_6", 0, 0, {}]]]}, {"name": "dropout_5", "config": {"name": "dropout_5", "rate": 0.25, "noise_shape": null, "seed": null, "trainable": true}, "class_name": "Dropout", "inbound_nodes": [[["up_sampling2d_3", 0, 0, {}]]]}], "name": "model_1", "input_layers": [["input_1", 0, 0]], "output_layers": [["dropout_5", 0, 0]]}, "class_name": "Model", "backend": "tensorflow"}
bin/pix2code2.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"config": {"name": "model_3", "layers": [{"name": "input_1", "config": {"dtype": "float32", "sparse": false, "batch_input_shape": [null, 256, 256, 3], "name": "input_1"}, "inbound_nodes": [], "class_name": "InputLayer"}, {"name": "model_2", "config": {"name": "model_2", "layers": [{"name": "input_1", "config": {"dtype": "float32", "sparse": false, "batch_input_shape": [null, 256, 256, 3], "name": "input_1"}, "inbound_nodes": [], "class_name": "InputLayer"}, {"name": "conv2d_1", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_1", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 32, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "conv2d_2", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_2", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 32, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["conv2d_1", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "max_pooling2d_1", "config": {"strides": [2, 2], "name": "max_pooling2d_1", "data_format": "channels_last", "trainable": false, "pool_size": [2, 2], "padding": "valid"}, "inbound_nodes": [[["conv2d_2", 0, 0, {}]]], "class_name": "MaxPooling2D"}, {"name": "dropout_1", "config": {"name": "dropout_1", "trainable": false, "noise_shape": null, "seed": null, "rate": 0.25}, "inbound_nodes": [[["max_pooling2d_1", 0, 0, {}]]], "class_name": "Dropout"}, {"name": "conv2d_3", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_3", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 64, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["dropout_1", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "conv2d_4", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_4", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 64, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["conv2d_3", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "max_pooling2d_2", "config": {"strides": [2, 2], "name": "max_pooling2d_2", "data_format": "channels_last", "trainable": false, "pool_size": [2, 2], "padding": "valid"}, "inbound_nodes": [[["conv2d_4", 0, 0, {}]]], "class_name": "MaxPooling2D"}, {"name": "dropout_2", "config": {"name": "dropout_2", "trainable": false, "noise_shape": null, "seed": null, "rate": 0.25}, "inbound_nodes": [[["max_pooling2d_2", 0, 0, {}]]], "class_name": "Dropout"}, {"name": "conv2d_5", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_5", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 128, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["dropout_2", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "conv2d_6", "config": {"kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "strides": [1, 1], "use_bias": true, "padding": "same", "bias_regularizer": null, "name": "conv2d_6", "data_format": "channels_last", "trainable": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "filters": 128, "kernel_size": [3, 3], "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "dilation_rate": [1, 1], "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["conv2d_5", 0, 0, {}]]], "class_name": "Conv2D"}, {"name": "max_pooling2d_3", "config": {"strides": [2, 2], "name": "max_pooling2d_3", "data_format": "channels_last", "trainable": false, "pool_size": [2, 2], "padding": "valid"}, "inbound_nodes": [[["conv2d_6", 0, 0, {}]]], "class_name": "MaxPooling2D"}, {"name": "encoded_layer", "config": {"name": "encoded_layer", "trainable": false, "noise_shape": null, "seed": null, "rate": 0.25}, "inbound_nodes": [[["max_pooling2d_3", 0, 0, {}]]], "class_name": "Dropout"}], "input_layers": [["input_1", 0, 0]], "output_layers": [["encoded_layer", 0, 0]]}, "inbound_nodes": [[["input_1", 0, 0, {}]]], "class_name": "Model"}, {"name": "flatten_1", "config": {"name": "flatten_1", "trainable": true}, "inbound_nodes": [[["model_2", 1, 0, {}]]], "class_name": "Flatten"}, {"name": "dense_1", "config": {"units": 1024, "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "use_bias": true, "bias_regularizer": null, "name": "dense_1", "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["flatten_1", 0, 0, {}]]], "class_name": "Dense"}, {"name": "dropout_6", "config": {"name": "dropout_6", "trainable": true, "noise_shape": null, "seed": null, "rate": 0.3}, "inbound_nodes": [[["dense_1", 0, 0, {}]]], "class_name": "Dropout"}, {"name": "dense_2", "config": {"units": 1024, "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "use_bias": true, "bias_regularizer": null, "name": "dense_2", "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "activity_regularizer": null, "activation": "relu"}, "inbound_nodes": [[["dropout_6", 0, 0, {}]]], "class_name": "Dense"}, {"name": "dropout_7", "config": {"name": "dropout_7", "trainable": true, "noise_shape": null, "seed": null, "rate": 0.3}, "inbound_nodes": [[["dense_2", 0, 0, {}]]], "class_name": "Dropout"}, {"name": "input_3", "config": {"dtype": "float32", "sparse": false, "batch_input_shape": [null, 48, 19], "name": "input_3"}, "inbound_nodes": [], "class_name": "InputLayer"}, {"name": "repeat_vector_1", "config": {"name": "repeat_vector_1", "trainable": true, "n": 48}, "inbound_nodes": [[["dropout_7", 0, 0, {}]]], "class_name": "RepeatVector"}, {"name": "sequential_1", "config": [{"config": {"recurrent_dropout": 0.0, "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "go_backwards": false, "use_bias": true, "bias_regularizer": null, "recurrent_initializer": {"config": {"gain": 1.0, "seed": null}, "class_name": "Orthogonal"}, "trainable": true, "stateful": false, "dtype": "float32", "recurrent_regularizer": null, "kernel_regularizer": null, "recurrent_constraint": null, "recurrent_activation": "hard_sigmoid", "units": 128, "batch_input_shape": [null, 48, 19], "name": "lstm_1", "unroll": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "dropout": 0.0, "implementation": 1, "return_state": false, "return_sequences": true, "bias_constraint": null, "unit_forget_bias": true, "kernel_constraint": null, "activity_regularizer": null, "activation": "tanh"}, "class_name": "LSTM"}, {"config": {"recurrent_dropout": 0.0, "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "go_backwards": false, "use_bias": true, "bias_regularizer": null, "recurrent_initializer": {"config": {"gain": 1.0, "seed": null}, "class_name": "Orthogonal"}, "trainable": true, "stateful": false, "recurrent_regularizer": null, "kernel_regularizer": null, "recurrent_constraint": null, "recurrent_activation": "hard_sigmoid", "units": 128, "name": "lstm_2", "unroll": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "dropout": 0.0, "implementation": 1, "return_state": false, "return_sequences": true, "bias_constraint": null, "unit_forget_bias": true, "kernel_constraint": null, "activity_regularizer": null, "activation": "tanh"}, "class_name": "LSTM"}], "inbound_nodes": [[["input_3", 0, 0, {}]]], "class_name": "Sequential"}, {"name": "concatenate_1", "config": {"name": "concatenate_1", "trainable": true, "axis": -1}, "inbound_nodes": [[["repeat_vector_1", 0, 0, {}], ["sequential_1", 1, 0, {}]]], "class_name": "Concatenate"}, {"name": "lstm_3", "config": {"recurrent_dropout": 0.0, "units": 512, "go_backwards": false, "use_bias": true, "bias_regularizer": null, "recurrent_initializer": {"config": {"gain": 1.0, "seed": null}, "class_name": "Orthogonal"}, "trainable": true, "recurrent_regularizer": null, "kernel_regularizer": null, "recurrent_constraint": null, "recurrent_activation": "hard_sigmoid", "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "kernel_constraint": null, "name": "lstm_3", "unroll": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "dropout": 0.0, "implementation": 1, "return_state": false, "return_sequences": true, "bias_constraint": null, "unit_forget_bias": true, "stateful": false, "activity_regularizer": null, "activation": "tanh"}, "inbound_nodes": [[["concatenate_1", 0, 0, {}]]], "class_name": "LSTM"}, {"name": "lstm_4", "config": {"recurrent_dropout": 0.0, "units": 512, "go_backwards": false, "use_bias": true, "bias_regularizer": null, "recurrent_initializer": {"config": {"gain": 1.0, "seed": null}, "class_name": "Orthogonal"}, "trainable": true, "recurrent_regularizer": null, "kernel_regularizer": null, "recurrent_constraint": null, "recurrent_activation": "hard_sigmoid", "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "kernel_constraint": null, "name": "lstm_4", "unroll": false, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "dropout": 0.0, "implementation": 1, "return_state": false, "return_sequences": false, "bias_constraint": null, "unit_forget_bias": true, "stateful": false, "activity_regularizer": null, "activation": "tanh"}, "inbound_nodes": [[["lstm_3", 0, 0, {}]]], "class_name": "LSTM"}, {"name": "dense_3", "config": {"units": 19, "kernel_initializer": {"config": {"scale": 1.0, "distribution": "uniform", "seed": null, "mode": "fan_avg"}, "class_name": "VarianceScaling"}, "use_bias": true, "bias_regularizer": null, "name": "dense_3", "trainable": true, "bias_initializer": {"config": {}, "class_name": "Zeros"}, "bias_constraint": null, "kernel_regularizer": null, "kernel_constraint": null, "activity_regularizer": null, "activation": "softmax"}, "inbound_nodes": [[["lstm_4", 0, 0, {}]]], "class_name": "Dense"}], "input_layers": [["input_1", 0, 0], ["input_3", 0, 0]], "output_layers": [["dense_3", 0, 0]]}, "keras_version": "2.1.2", "backend": "tensorflow", "class_name": "Model"}
bin/words.vocab ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <START>-> 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
2
+ ,-> 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
3
+ {-> 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
4
+ -> 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
5
+ header-> 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
6
+ btn-active-> 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
7
+
8
+ -> 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
9
+ text-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.
10
+ quadruple-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.
11
+ btn-inactive-> 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
12
+ }-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.
13
+ btn-orange-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.
14
+ small-title-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.
15
+ <END>-> 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
16
+ double-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.
17
+ btn-red-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.
18
+ row-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.
19
+ single-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.
20
+ btn-green-> 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.
classes/.DS_Store ADDED
Binary file (6.15 kB). View file
 
classes/Sampler.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import print_function
2
+ from __future__ import absolute_import
3
+ __author__ = 'Tony Beltramelli - www.tonybeltramelli.com'
4
+
5
+ from .Vocabulary import *
6
+ # from .BeamSearch import *
7
+ from .Utils import *
8
+
9
+
10
+ class Sampler:
11
+ def __init__(self, voc_path, input_shape, output_size, context_length):
12
+ self.voc = Vocabulary()
13
+ self.voc.retrieve(voc_path)
14
+
15
+ self.input_shape = input_shape
16
+ self.output_size = output_size
17
+
18
+ print("Vocabulary size: {}".format(self.voc.size))
19
+ print("Input shape: {}".format(self.input_shape))
20
+ print("Output size: {}".format(self.output_size))
21
+
22
+ self.context_length = context_length
23
+
24
+ def predict_greedy(self, model, input_img, require_sparse_label=True, sequence_length=150, verbose=False):
25
+ current_context = [self.voc.vocabulary[PLACEHOLDER]] * (self.context_length - 1)
26
+ current_context.append(self.voc.vocabulary[START_TOKEN])
27
+ if require_sparse_label:
28
+ current_context = Utils.sparsify(current_context, self.output_size)
29
+
30
+ predictions = START_TOKEN
31
+ out_probas = []
32
+
33
+ for i in range(0, sequence_length):
34
+ if verbose:
35
+ print("predicting {}/{}...".format(i, sequence_length))
36
+
37
+ probas = model.predict(input_img, np.array([current_context]))
38
+ prediction = np.argmax(probas)
39
+ out_probas.append(probas)
40
+
41
+ new_context = []
42
+ for j in range(1, self.context_length):
43
+ new_context.append(current_context[j])
44
+
45
+ if require_sparse_label:
46
+ sparse_label = np.zeros(self.output_size)
47
+ sparse_label[prediction] = 1
48
+ new_context.append(sparse_label)
49
+ else:
50
+ new_context.append(prediction)
51
+
52
+ current_context = new_context
53
+
54
+ predictions += self.voc.token_lookup[prediction]
55
+
56
+ if self.voc.token_lookup[prediction] == END_TOKEN:
57
+ break
58
+
59
+ return predictions, out_probas
60
+
61
+ # def recursive_beam_search(self, model, input_img, current_context, beam, current_node, sequence_length):
62
+ # probas = model.predict(input_img, np.array([current_context]))
63
+
64
+ # predictions = []
65
+ # for i in range(0, len(probas)):
66
+ # predictions.append((i, probas[i], probas))
67
+
68
+ # nodes = []
69
+ # for i in range(0, len(predictions)):
70
+ # prediction = predictions[i][0]
71
+ # score = predictions[i][1]
72
+ # output_probas = predictions[i][2]
73
+ # nodes.append(Node(prediction, score, output_probas))
74
+
75
+ # beam.add_nodes(current_node, nodes)
76
+
77
+ # if beam.is_valid():
78
+ # beam.prune_leaves()
79
+ # if sequence_length == 1 or self.voc.token_lookup[beam.root.max_child().key] == END_TOKEN:
80
+ # return
81
+
82
+ # for node in beam.get_leaves():
83
+ # prediction = node.key
84
+
85
+ # new_context = []
86
+ # for j in range(1, self.context_length):
87
+ # new_context.append(current_context[j])
88
+ # sparse_label = np.zeros(self.output_size)
89
+ # sparse_label[prediction] = 1
90
+ # new_context.append(sparse_label)
91
+
92
+ # self.recursive_beam_search(model, input_img, new_context, beam, node, sequence_length - 1)
93
+
94
+ # def predict_beam_search(self, model, input_img, beam_width=3, require_sparse_label=True, sequence_length=150):
95
+ # predictions = START_TOKEN
96
+ # out_probas = []
97
+
98
+ # current_context = [self.voc.vocabulary[PLACEHOLDER]] * (self.context_length - 1)
99
+ # current_context.append(self.voc.vocabulary[START_TOKEN])
100
+ # if require_sparse_label:
101
+ # current_context = Utils.sparsify(current_context, self.output_size)
102
+
103
+ # beam = BeamSearch(beam_width=beam_width)
104
+
105
+ # self.recursive_beam_search(model, input_img, current_context, beam, beam.root, sequence_length)
106
+
107
+ # predicted_sequence, probas_sequence = beam.search()
108
+
109
+ # for k in range(0, len(predicted_sequence)):
110
+ # prediction = predicted_sequence[k]
111
+ # probas = probas_sequence[k]
112
+ # out_probas.append(probas)
113
+
114
+ # predictions += self.voc.token_lookup[prediction]
115
+
116
+ # return predictions, out_probas
classes/Utils.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+
3
+
4
+ class Utils:
5
+ @staticmethod
6
+ def sparsify(label_vector, output_size):
7
+ sparse_vector = []
8
+
9
+ for label in label_vector:
10
+ sparse_label = np.zeros(output_size)
11
+ sparse_label[label] = 1
12
+
13
+ sparse_vector.append(sparse_label)
14
+
15
+ return np.array(sparse_vector)
16
+
17
+ @staticmethod
18
+ def get_preprocessed_img(img_path, image_size):
19
+ import cv2
20
+ # from keras.preprocessing.image import array_to_img, img_to_array
21
+ # img = array_to_img(img_path)
22
+ # img = img_to_array(img)
23
+ # img = cv2.imread(img_path)
24
+ # don't need to read the image as we're now directly passing the
25
+ # image as numpy array to this method
26
+ img = cv2.resize(img_path, (image_size, image_size))
27
+ img = img.astype('float32')
28
+ img /= 255
29
+ return img
30
+
31
+ @staticmethod
32
+ def show(image):
33
+ import cv2
34
+ cv2.namedWindow("view", cv2.WINDOW_AUTOSIZE)
35
+ cv2.imshow("view", image)
36
+ cv2.waitKey(0)
37
+ cv2.destroyWindow("view")
classes/Vocabulary.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __author__ = 'Tony Beltramelli - www.tonybeltramelli.com'
2
+
3
+ import sys
4
+ import numpy as np
5
+
6
+ START_TOKEN = "<START>"
7
+ END_TOKEN = "<END>"
8
+ PLACEHOLDER = " "
9
+ SEPARATOR = '->'
10
+
11
+
12
+ class Vocabulary:
13
+ def __init__(self):
14
+ self.binary_vocabulary = {}
15
+ self.vocabulary = {}
16
+ self.token_lookup = {}
17
+ self.size = 0
18
+
19
+ self.append(START_TOKEN)
20
+ self.append(END_TOKEN)
21
+ self.append(PLACEHOLDER)
22
+
23
+ def append(self, token):
24
+ if token not in self.vocabulary:
25
+ self.vocabulary[token] = self.size
26
+ self.token_lookup[self.size] = token
27
+ self.size += 1
28
+
29
+ def create_binary_representation(self):
30
+ if sys.version_info >= (3,):
31
+ items = self.vocabulary.items()
32
+ else:
33
+ items = self.vocabulary.iteritems()
34
+ for key, value in items:
35
+ binary = np.zeros(self.size)
36
+ binary[value] = 1
37
+ self.binary_vocabulary[key] = binary
38
+
39
+ def get_serialized_binary_representation(self):
40
+ if len(self.binary_vocabulary) == 0:
41
+ self.create_binary_representation()
42
+
43
+ string = ""
44
+ if sys.version_info >= (3,):
45
+ items = self.binary_vocabulary.items()
46
+ else:
47
+ items = self.binary_vocabulary.iteritems()
48
+ for key, value in items:
49
+ array_as_string = np.array2string(value, separator=',', max_line_width=self.size * self.size)
50
+ string += "{}{}{}\n".format(key, SEPARATOR, array_as_string[1:len(array_as_string) - 1])
51
+ return string
52
+
53
+ def save(self, path):
54
+ output_file_name = "{}/words.vocab".format(path)
55
+ output_file = open(output_file_name, 'w')
56
+ output_file.write(self.get_serialized_binary_representation())
57
+ output_file.close()
58
+
59
+ def retrieve(self, path):
60
+ input_file = open("{}/words.vocab".format(path), 'r')
61
+ buffer = ""
62
+ for line in input_file:
63
+ try:
64
+ separator_position = len(buffer) + line.index(SEPARATOR)
65
+ buffer += line
66
+ key = buffer[:separator_position]
67
+ value = buffer[separator_position + len(SEPARATOR):]
68
+ value = np.fromstring(value, sep=',')
69
+
70
+ self.binary_vocabulary[key] = value
71
+ self.vocabulary[key] = np.where(value == 1)[0][0]
72
+ self.token_lookup[np.where(value == 1)[0][0]] = key
73
+
74
+ buffer = ""
75
+ except ValueError:
76
+ buffer += line
77
+ input_file.close()
78
+ self.size = len(self.vocabulary)
classes/__init__.py ADDED
File without changes
classes/__pycache__/BeamSearch.cpython-35.pyc ADDED
Binary file (4.56 kB). View file
 
classes/__pycache__/BeamSearch.cpython-38.pyc ADDED
Binary file (4.2 kB). View file
 
classes/__pycache__/BeamSearch.cpython-39.pyc ADDED
Binary file (4.23 kB). View file
 
classes/__pycache__/Sampler.cpython-35.pyc ADDED
Binary file (3.39 kB). View file
 
classes/__pycache__/Sampler.cpython-38.pyc ADDED
Binary file (1.76 kB). View file
 
classes/__pycache__/Sampler.cpython-39.pyc ADDED
Binary file (3.09 kB). View file
 
classes/__pycache__/Utils.cpython-35.pyc ADDED
Binary file (1.28 kB). View file
 
classes/__pycache__/Utils.cpython-38.pyc ADDED
Binary file (1.19 kB). View file
 
classes/__pycache__/Utils.cpython-39.pyc ADDED
Binary file (1.24 kB). View file
 
classes/__pycache__/Vocabulary.cpython-35.pyc ADDED
Binary file (2.86 kB). View file
 
classes/__pycache__/Vocabulary.cpython-38.pyc ADDED
Binary file (2.61 kB). View file
 
classes/__pycache__/Vocabulary.cpython-39.pyc ADDED
Binary file (2.64 kB). View file
 
classes/__pycache__/__init__.cpython-35.pyc ADDED
Binary file (150 Bytes). View file
 
classes/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (163 Bytes). View file
 
classes/__pycache__/__init__.cpython-39.pyc ADDED
Binary file (188 Bytes). View file
 
classes/model/AModel.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __author__ = 'Tony Beltramelli - www.tonybeltramelli.com'
2
+
3
+ from keras.models import model_from_json
4
+
5
+
6
+ class AModel:
7
+ def __init__(self, input_shape, output_size, output_path):
8
+ self.model = None
9
+ self.input_shape = input_shape
10
+ self.output_size = output_size
11
+ self.output_path = output_path
12
+ self.name = ""
13
+
14
+ def save(self):
15
+ model_json = self.model.to_json()
16
+ with open("{}/{}.json".format(self.output_path, self.name), "w") as json_file:
17
+ json_file.write(model_json)
18
+ self.model.save_weights("{}/{}.h5".format(self.output_path, self.name))
19
+
20
+ def load(self, name=""):
21
+ output_name = self.name if name == "" else name
22
+ with open("{}/{}.json".format(self.output_path, output_name), "r") as json_file:
23
+ loaded_model_json = json_file.read()
24
+ self.model = model_from_json(loaded_model_json)
25
+ self.model.load_weights("{}/{}.h5".format(self.output_path, output_name))
classes/model/Config.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ __author__ = 'Tony Beltramelli - www.tonybeltramelli.com'
2
+
3
+ CONTEXT_LENGTH = 48
4
+ IMAGE_SIZE = 256
5
+ BATCH_SIZE = 64
6
+ EPOCHS = 10
7
+ STEPS_PER_EPOCH = 72000
classes/model/__init__.py ADDED
File without changes
classes/model/__pycache__/AModel.cpython-35.pyc ADDED
Binary file (1.36 kB). View file
 
classes/model/__pycache__/AModel.cpython-38.pyc ADDED
Binary file (1.28 kB). View file
 
classes/model/__pycache__/Config.cpython-35.pyc ADDED
Binary file (343 Bytes). View file
 
classes/model/__pycache__/Config.cpython-38.pyc ADDED
Binary file (344 Bytes). View file
 
classes/model/__pycache__/__init__.cpython-35.pyc ADDED
Binary file (156 Bytes). View file
 
classes/model/__pycache__/__init__.cpython-38.pyc ADDED
Binary file (169 Bytes). View file
 
classes/model/__pycache__/__init__.cpython-39.pyc ADDED
Binary file (194 Bytes). View file
 
classes/model/__pycache__/autoencoder_image.cpython-35.pyc ADDED
Binary file (2.58 kB). View file
 
classes/model/__pycache__/autoencoder_image.cpython-38.pyc ADDED
Binary file (2.28 kB). View file
 
classes/model/__pycache__/pix2code.cpython-35.pyc ADDED
Binary file (3.31 kB). View file
 
classes/model/__pycache__/pix2code2.cpython-35.pyc ADDED
Binary file (2.83 kB). View file
 
classes/model/__pycache__/pix2code2.cpython-38.pyc ADDED
Binary file (2.61 kB). View file
 
classes/model/__pycache__/pix2code2.cpython-39.pyc ADDED
Binary file (2.63 kB). View file
 
classes/model/autoencoder_image.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __author__ = 'Ferdinand John Briones, attempt at pix2code2 through pretrained autoencoders'
2
+
3
+ from keras.layers import Input, Dropout, Conv2D, MaxPooling2D, Flatten, Conv2DTranspose, UpSampling2D, Reshape, Dense
4
+ from keras.models import Sequential, Model
5
+ # from keras.optimizers import RMSprop
6
+ from tensorflow.keras.optimizers import RMSprop
7
+ from keras import *
8
+ from .Config import *
9
+ from .AModel import *
10
+
11
+
12
+ class autoencoder_image(AModel):
13
+ def __init__(self, input_shape, output_size, output_path):
14
+ AModel.__init__(self, input_shape, output_size, output_path)
15
+ self.name = 'autoencoder'
16
+
17
+ input_image = Input(shape=input_shape)
18
+ encoder = Conv2D(32, 3, padding='same', activation='relu')(input_image)
19
+ encoder = Conv2D(32, 3, padding='same', activation='relu')(encoder)
20
+ encoder = MaxPooling2D()(encoder)
21
+ encoder = Dropout(0.25)(encoder)
22
+
23
+ encoder = Conv2D(64, 3, padding='same', activation='relu')(encoder)
24
+ encoder = Conv2D(64, 3, padding='same', activation='relu')(encoder)
25
+ encoder = MaxPooling2D()(encoder)
26
+ encoder = Dropout(0.25)(encoder)
27
+
28
+ encoder = Conv2D(128, 3, padding='same', activation='relu')(encoder)
29
+ encoder = Conv2D(128, 3, padding='same', activation='relu')(encoder)
30
+ encoder = MaxPooling2D()(encoder)
31
+ encoded = Dropout(0.25, name='encoded_layer')(encoder)
32
+
33
+ decoder = Conv2DTranspose(128, 3, padding='same', activation='relu')(encoded)
34
+ decoder = Conv2DTranspose(128, 3, padding='same', activation='relu')(decoder)
35
+ decoder = UpSampling2D()(decoder)
36
+ decoder = Dropout(0.25)(decoder)
37
+
38
+ decoder = Conv2DTranspose(64, 3, padding='same', activation='relu')(decoder)
39
+ decoder = Conv2DTranspose(64, 3, padding='same', activation='relu')(decoder)
40
+ decoder = UpSampling2D()(decoder)
41
+ decoder = Dropout(0.25)(decoder)
42
+
43
+ decoder = Conv2DTranspose(32, 3, padding='same', activation='relu')(decoder)
44
+ decoder = Conv2DTranspose(3, 3, padding='same', activation='relu')(decoder)
45
+ decoder = UpSampling2D()(decoder)
46
+ decoded = Dropout(0.25)(decoder)
47
+
48
+ # decoder = Dense(256*256*3)(decoder)
49
+ # decoded = Reshape(target_shape=input_shape)(decoder)
50
+
51
+ self.model = Model(input_image, decoded)
52
+ self.model.compile(optimizer='adadelta', loss='binary_crossentropy')
53
+ self.model.summary()
54
+
55
+ def fit_generator(self, generator, steps_per_epoch):
56
+ self.model.fit_generator(generator, steps_per_epoch=steps_per_epoch, epochs=EPOCHS, verbose=1)
57
+ self.save()
58
+
59
+ def predict_hidden(self, images):
60
+ hidden_layer_model = Model(inputs = self.input, outputs = self.get_layer('encoded_layer').output)
61
+ return hidden_layer_model.predict(images)
classes/model/pix2code2.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __author__ = 'Ferdiand John Briones, attempt at pix2code2 through pretrained autoencoders'
2
+
3
+ from keras.layers import Input, Dense, Dropout, RepeatVector, LSTM, concatenate, Flatten
4
+ from keras.models import Sequential, Model
5
+ from tensorflow.keras.optimizers import RMSprop
6
+ from keras import *
7
+ from .Config import *
8
+ from .AModel import *
9
+ from .autoencoder_image import *
10
+
11
+ class pix2code2(AModel):
12
+ def __init__(self, input_shape, output_size, output_path):
13
+ AModel.__init__(self, input_shape, output_size, output_path)
14
+ self.name = "pix2code2"
15
+
16
+ visual_input = Input(shape=input_shape)
17
+
18
+ #Load the pre-trained autoencoder model
19
+ autoencoder_model = autoencoder_image(input_shape, input_shape, output_path)
20
+ autoencoder_model.load('autoencoder')
21
+ autoencoder_model.model.load_weights('../bin/autoencoder.h5')
22
+
23
+ #Get only the model up to the encoded part
24
+ hidden_layer_model_freeze = Model(inputs=autoencoder_model.model.input, outputs=autoencoder_model.model.get_layer('encoded_layer').output)
25
+ hidden_layer_input = hidden_layer_model_freeze(visual_input)
26
+
27
+ #Additional layers before concatenation
28
+ hidden_layer_model = Flatten()(hidden_layer_input)
29
+ hidden_layer_model = Dense(1024, activation='relu')(hidden_layer_model)
30
+ hidden_layer_model = Dropout(0.3)(hidden_layer_model)
31
+ hidden_layer_model = Dense(1024, activation='relu')(hidden_layer_model)
32
+ hidden_layer_model = Dropout(0.3)(hidden_layer_model)
33
+ hidden_layer_result = RepeatVector(CONTEXT_LENGTH)(hidden_layer_model)
34
+
35
+ #Make sure the loaded hidden_layer_model_freeze will no longer be updated
36
+ for layer in hidden_layer_model_freeze.layers:
37
+ layer.trainable = False
38
+
39
+ #The same language model that of pix2code by Tony Beltramelli
40
+ language_model = Sequential()
41
+ language_model.add(LSTM(128, return_sequences=True, input_shape=(CONTEXT_LENGTH, output_size)))
42
+ language_model.add(LSTM(128, return_sequences=True))
43
+
44
+ textual_input = Input(shape=(CONTEXT_LENGTH, output_size))
45
+ encoded_text = language_model(textual_input)
46
+
47
+ decoder = concatenate([hidden_layer_result, encoded_text])
48
+
49
+ decoder = LSTM(512, return_sequences=True)(decoder)
50
+ decoder = LSTM(512, return_sequences=False)(decoder)
51
+ decoder = Dense(output_size, activation='softmax')(decoder)
52
+
53
+ self.model = Model(inputs=[visual_input, textual_input], outputs=decoder)
54
+
55
+ optimizer = RMSprop(lr=0.0001, clipvalue=1.0)
56
+ self.model.compile(loss='categorical_crossentropy', optimizer=optimizer)
57
+
58
+ def fit_generator(self, generator, steps_per_epoch):
59
+ self.model.summary()
60
+ self.model.fit_generator(generator, steps_per_epoch=steps_per_epoch, epochs=EPOCHS, verbose=1)
61
+ self.save()
62
+
63
+ def predict(self, image, partial_caption):
64
+ return self.model.predict([image, partial_caption], verbose=0)[0]
65
+
66
+ def predict_batch(self, images, partial_captions):
67
+ return self.model.predict([images, partial_captions], verbose=1)
data/.DS_Store ADDED
Binary file (6.15 kB). View file
 
main_program.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import absolute_import
2
+ from __future__ import print_function
3
+
4
+ import os.path
5
+ from os.path import basename
6
+
7
+ from classes.Sampler import *
8
+ from classes.model.pix2code2 import *
9
+
10
+
11
+ def code_gen(path_to_input_image):
12
+ # if len(sys.argv) < 1:
13
+ # print("Error: not enough argument supplied:")
14
+ # # print( "sample.py <trained weights path> <trained model name> <input image> <output path> <search method (
15
+ # # default: " "greedy)>")
16
+ # exit(0)
17
+ # else:
18
+ trained_weights_path = "../bin"
19
+ trained_model_name = "pix2code2"
20
+ # input_path = sys.argv[1]
21
+ # input_path = "../data/test_gu.png"
22
+ input_path = path_to_input_image
23
+ output_path = "../data/output/"
24
+ search_method = "greedy"
25
+
26
+ meta_dataset = np.load("{}/meta_dataset.npy".format(trained_weights_path), allow_pickle=True)
27
+ input_shape = meta_dataset[0]
28
+ output_size = meta_dataset[1]
29
+
30
+ model = pix2code2(input_shape, output_size, trained_weights_path)
31
+ model.load(trained_model_name)
32
+
33
+ sampler = Sampler(trained_weights_path, input_shape, output_size, CONTEXT_LENGTH)
34
+
35
+ file_name = 'input_image_from_interface.png'
36
+ file_name = basename(file_name)[:basename(file_name).find(".")]
37
+ evaluation_img = Utils.get_preprocessed_img(input_path, IMAGE_SIZE)
38
+
39
+ if search_method == "greedy":
40
+ result, _ = sampler.predict_greedy(model, np.array([evaluation_img]))
41
+ print("Result greedy: \n {}".format(result))
42
+ # else:
43
+ # beam_width = int(search_method)
44
+ # print("Search with beam width: {}".format(beam_width))
45
+ # result, _ = sampler.predict_beam_search(model, np.array([evaluation_img]), beam_width=beam_width)
46
+ # print("Result beam: {}".format(result))
47
+
48
+ with open("{}/{}.gui".format(output_path, file_name), 'w') as out_f:
49
+ out_f.write(result.replace(START_TOKEN, "").replace(END_TOKEN, ""))
50
+
51
+ return output_path, file_name
52
+
53
+
54
+ def compile_gui(outputpath, filename):
55
+ from os.path import basename
56
+ from compclasses.Utils import Utils
57
+ from compclasses.Compiler import Compiler
58
+
59
+ input_path = (outputpath+filename)
60
+
61
+ # remove the path
62
+ file_ = os.path.basename(input_path)
63
+ # remove the extension
64
+ file_ = os.path.splitext(file_)[0]
65
+ # add the extension of gui
66
+ file_ = "../data/output/" + file_ + ".gui"
67
+
68
+ input_file = file_
69
+
70
+ FILL_WITH_RANDOM_TEXT = True
71
+ TEXT_PLACE_HOLDER = "[]"
72
+
73
+ dsl_path = "assets/web-dsl-mapping.json"
74
+ compiler = Compiler(dsl_path)
75
+
76
+ def render_content_with_text(key, value):
77
+ if FILL_WITH_RANDOM_TEXT:
78
+ if key.find("btn") != -1:
79
+ value = value.replace(TEXT_PLACE_HOLDER, Utils.get_random_text())
80
+ elif key.find("title") != -1:
81
+ value = value.replace(TEXT_PLACE_HOLDER, Utils.get_random_text(length_text=5, space_number=0))
82
+ elif key.find("text") != -1:
83
+ value = value.replace(TEXT_PLACE_HOLDER,
84
+ Utils.get_random_text(length_text=56, space_number=7, with_upper_case=False))
85
+ return value
86
+
87
+ file_uid = basename(input_file)[:basename(input_file).find(".")]
88
+ path = input_file[:input_file.find(file_uid)]
89
+
90
+ input_file_path = "{}{}.gui".format(path, file_uid)
91
+ output_file_path = "{}{}.html".format(path, file_uid)
92
+
93
+ html_code = compiler.compile(input_file_path, output_file_path, rendering_function=render_content_with_text)
94
+ print("Generated code is compiled..!!")
95
+ return html_code
96
+
97
+
98
+ def main_program(path_to_file):
99
+ output_path, file_name = code_gen(path_to_file)
100
+ result = compile_gui(output_path, file_name)
101
+ return result