Repository: piskvorky/gensim

Issue:
Investigate and fix Keras problem under Python 3.8
One of our unit tests fails on Travis under Py3.8.

```
=================================== FAILURES ===================================

_____________ TestKerasWord2VecWrapper.testEmbeddingLayerCosineSim _____________

self = <gensim.test.test_keras_integration.TestKerasWord2VecWrapper testMethod=testEmbeddingLayerCosineSim>

    def testEmbeddingLayerCosineSim(self):

        """

        Test Keras 'Embedding' layer returned by 'get_embedding_layer' function for a simple word similarity task.

        """

        keras_w2v_model = self.model_cos_sim

        keras_w2v_model_wv = keras_w2v_model.wv

    

        embedding_layer = keras_w2v_model_wv.get_keras_embedding()

    

        input_a = Input(shape=(1,), dtype='int32', name='input_a')

        input_b = Input(shape=(1,), dtype='int32', name='input_b')

        embedding_a = embedding_layer(input_a)

        embedding_b = embedding_layer(input_b)

        similarity = dot([embedding_a, embedding_b], axes=2, normalize=True)

    

>       model = Model(input=[input_a, input_b], output=similarity)

embedding_a = <tf.Tensor 'embedding_4/Identity:0' shape=(None, 1, 100) dtype=float32>

embedding_b = <tf.Tensor 'embedding_4_1/Identity:0' shape=(None, 1, 100) dtype=float32>

embedding_layer = <tensorflow.python.keras.layers.embeddings.Embedding object at 0x7f603df0d130>

input_a    = <tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>

input_b    = <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>

keras_w2v_model = <gensim.models.word2vec.Word2Vec object at 0x7f603df0d760>

keras_w2v_model_wv = <gensim.models.keyedvectors.Word2VecKeyedVectors object at 0x7f603df0d250>

self       = <gensim.test.test_keras_integration.TestKerasWord2VecWrapper testMethod=testEmbeddingLayerCosineSim>

similarity = <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>

gensim/test/test_keras_integration.py:62: 

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:167: in __init__

    super(Model, self).__init__(*args, **kwargs)

        __class__  = <class 'tensorflow.python.keras.engine.training.Model'>

        args       = ()

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/keras/engine/network.py:176: in __init__

    self._init_subclassed_network(**kwargs)

        args       = ()

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/training/tracking/base.py:456: in _method_wrapper

    result = method(self, *args, **kwargs)

        args       = ()

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        method     = <function Network._init_subclassed_network at 0x7f60465f6d30>

        previous_value = True

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/keras/engine/network.py:367: in _init_subclassed_network

    self._base_init(name=name, **kwargs)

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        name       = None

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/training/tracking/base.py:456: in _method_wrapper

    result = method(self, *args, **kwargs)

        args       = ()

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'name': None, 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        method     = <function Network._base_init at 0x7f60465f6a60>

        previous_value = False

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

.tox/py38-linux/lib/python3.8/site-packages/tensorflow/python/keras/engine/network.py:202: in _base_init

    generic_utils.validate_kwargs(kwargs, {'trainable', 'dtype', 'dynamic',

        __class__  = <class 'tensorflow.python.keras.engine.network.Network'>

        kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

        name       = None

        self       = <tensorflow.python.keras.engine.training.Model object at 0x7f603df18ac0>

_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

kwargs = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}

allowed_kwargs = {'autocast', 'dtype', 'dynamic', 'trainable'}

error_message = 'Keyword argument not understood:'

    def validate_kwargs(kwargs,

                        allowed_kwargs,

                        error_message='Keyword argument not understood:'):

      """Checks that all keyword arguments are in the set of allowed keys."""

      for kwarg in kwargs:

        if kwarg not in allowed_kwargs:

>         raise TypeError(error_message, kwarg)

E         TypeError: ('Keyword argument not understood:', 'input')

allowed_kwargs = {'autocast', 'dtype', 'dynamic', 'trainable'}

error_message = 'Keyword argument not understood:'

kwarg      = 'input'

kwargs     = {'input': [<tf.Tensor 'input_a_3:0' shape=(None, 1) dtype=int32>, <tf.Tensor 'input_b_3:0' shape=(None, 1) dtype=int32>], 'output': <tf.Tensor 'dot_3/Identity:0' shape=(None, 1, 1) dtype=float32>}
```


Discussion:
Also, another test in the same class was also failing a type assertion: 

```
=================================== FAILURES ===================================
____________ TestKerasWord2VecWrapper.testEmbeddingLayer20NewsGroup ____________
self = <gensim.test.test_keras_integration.TestKerasWord2VecWrapper testMethod=testEmbeddingLayer20NewsGroup>
    def testEmbeddingLayer20NewsGroup(self):
        """
        Test Keras 'Embedding' layer returned by 'get_embedding_layer' function
        for a smaller version of the 20NewsGroup classification problem.
        """
        MAX_SEQUENCE_LENGTH = 1000
    
        # Prepare text samples and their labels
    
        # Processing text dataset
        texts = []  # list of text samples
        texts_w2v = []  # used to train the word embeddings
        labels = []  # list of label ids
    
        data = fetch_20newsgroups(subset='train', categories=['alt.atheism', 'comp.graphics', 'sci.space'])
        for index in range(len(data)):
            label_id = data.target[index]
            file_data = data.data[index]
            i = file_data.find('\n\n')  # skip header
            if i > 0:
                file_data = file_data[i:]
            try:
                curr_str = str(file_data)
                sentence_list = curr_str.split('\n')
                for sentence in sentence_list:
                    sentence = (sentence.strip()).lower()
                    texts.append(sentence)
                    texts_w2v.append(sentence.split(' '))
                    labels.append(label_id)
            except Exception:
                pass
    
        # Vectorize the text samples into a 2D integer tensor
        tokenizer = Tokenizer()
        tokenizer.fit_on_texts(texts)
        sequences = tokenizer.texts_to_sequences(texts)
    
        # word_index = tokenizer.word_index
        data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
        labels = to_categorical(np.asarray(labels))
    
        x_train = data
        y_train = labels
    
        # prepare the embedding layer using the wrapper
        keras_w2v = self.model_twenty_ng
        keras_w2v.build_vocab(texts_w2v)
        keras_w2v.train(texts, total_examples=keras_w2v.corpus_count, epochs=keras_w2v.epochs)
        keras_w2v_wv = keras_w2v.wv
        embedding_layer = keras_w2v_wv.get_keras_embedding()
    
        # create a 1D convnet to solve our classification task
        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
        embedded_sequences = embedding_layer(sequence_input)
        x = Conv1D(128, 5, activation='relu')(embedded_sequences)
        x = MaxPooling1D(5)(x)
        x = Conv1D(128, 5, activation='relu')(x)
        x = MaxPooling1D(5)(x)
        x = Conv1D(128, 5, activation='relu')(x)
        x = MaxPooling1D(35)(x)  # global max pooling
        x = Flatten()(x)
        x = Dense(128, activation='relu')(x)
        preds = Dense(y_train.shape[1], activation='softmax')(x)
    
        model = Model(sequence_input, preds)
        model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
        fit_ret_val = model.fit(x_train, y_train, epochs=1)
    
        # verify the type of the object returned after training
        # value returned is a `History` instance.
        # Its `history` attribute contains all information collected during training.
>       self.assertTrue(type(fit_ret_val) == keras.callbacks.History)
E       AssertionError: False is not true
MAX_SEQUENCE_LENGTH = 1000
curr_str   = '\n\nIn article <bill.047m@xpresso.UUCP> bill@xpresso.UUCP (Bill Vance) writes:\n>It has been known for quite a while ... Henry Spencer @ U of Toronto Zoology\nbetween SVR3 and SunOS.    - Dick Dunn  |  henry@zoo.toronto.edu  utzoo!henry\n'
data       = array([[  0,   0,   0, ...,   0,   0,   0],
       [  0,   0,   0, ...,   0,   0,   0],
       [  0,   0,   0, ...,   ...,   5,  89, 360],
       [  0,   0,   0, ..., 366, 367,  42],
       [  0,   0,   0, ...,   0,   0,   0]], dtype=int32)
embedded_sequences = <tf.Tensor 'embedding_3/Identity:0' shape=(None, 1000, 100) dtype=float32>
embedding_layer = <tensorflow.python.keras.layers.embeddings.Embedding object at 0x7f28eebbbe50>
file_data  = '\n\nIn article <bill.047m@xpresso.UUCP> bill@xpresso.UUCP (Bill Vance) writes:\n>It has been known for quite a while ... Henry Spencer @ U of Toronto Zoology\nbetween SVR3 and SunOS.    - Dick Dunn  |  henry@zoo.toronto.edu  utzoo!henry\n'
fit_ret_val = <tensorflow.python.keras.callbacks.History object at 0x7f28ec0a18b0>
i          = 129
index      = 4
keras_w2v  = <gensim.models.word2vec.Word2Vec object at 0x7f28fd910a30>
keras_w2v_wv = <gensim.models.keyedvectors.Word2VecKeyedVectors object at 0x7f28fd910190>
label_id   = 2
labels     = array([[0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0...      [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.]], dtype=float32)
model      = <tensorflow.python.keras.engine.training.Model object at 0x7f28ec0ed490>
preds      = <tf.Tensor 'dense_7/Identity:0' shape=(None, 3) dtype=float32>
self       = <gensim.test.test_keras_integration.TestKerasWord2VecWrapper testMethod=testEmbeddingLayer20NewsGroup>
sentence   = ''
sentence_list = ['', '', 'In article <bill.047m@xpresso.UUCP> bill@xpresso.UUCP (Bill Vance) writes:', '>It has been known for quite a... Does anyone make a "globe" that is accurate', '>as to actual shape, landmass configuration/Long/Lat lines etc.?', ...]
sequence_input = <tf.Tensor 'input_4:0' shape=(None, 1000) dtype=int32>
sequences  = [[], [], [], [10, 90, 11, 12, 43, 91, ...], [13, 95, 96, 44, 97, 98, ...], [100, 5, 101, 102, 103, 4, ...], ...]
texts      = ['', '', '', 'i doubt there are good prospects for  a self armoring system', 'for venus surface conditions (several hundred degrees, very high', 'pressure of co2, possibly sulfuric and nitric acids or oxides', ...]
texts_w2v  = [[''], [''], [''], ['i', 'doubt', 'there', 'are', 'good', 'prospects', ...], ['for', 'venus', 'surface', 'conditions', '(several', 'hundred', ...], ['pressure', 'of', 'co2,', 'possibly', 'sulfuric', 'and', ...], ...]
tokenizer  = <keras_preprocessing.text.Tokenizer object at 0x7f28fd9ad910>
x          = <tf.Tensor 'dense_6/Identity:0' shape=(None, 128) dtype=float32>
x_train    = array([[  0,   0,   0, ...,   0,   0,   0],
       [  0,   0,   0, ...,   0,   0,   0],
       [  0,   0,   0, ...,   ...,   5,  89, 360],
       [  0,   0,   0, ..., 366, 367,  42],
       [  0,   0,   0, ...,   0,   0,   0]], dtype=int32)
y_train    = array([[0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0...      [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.],
       [0., 0., 1.]], dtype=float32)
gensim/test/test_keras_integration.py:146: AssertionError
----------------------------- Captured stdout call -----------------------------
3/3 [==============================] - 0s 143ms/step - loss: 1.0621 - acc: 0.4688
----------------------------- Captured stderr call -----------------------------
```
Per my [comment on the version-pinning workaround](https://github.com/RaRe-Technologies/gensim/pull/2868#issuecomment-650347295):

> Given that it is likely a keras change, and others may be bitten by it and get them to ensure py3.8 works the same as earlier Python in due time, I'd put full resolution of #2865 into a "wait and see if somebody else fixes the issue" bin. By the time gensim needs to do a release, we could (1) see if by then, simply removing the pin no longer causes any problems; (2) leave the pin & add a brief release note "stay with keras<3.4 if on Python 3.8" (which might be something users on Py3.8 would need to do anyway); or finally and least-desirably: (3) actually dig into the keras change & fix on gensim's end.
Archiving my https://github.com/RaRe-Technologies/gensim/pull/2868#issuecomment-650333911 too:

> Quick googling of the error message `Keyword argument not understood` suggests that Keras expects parameters named `inputs` and `outputs`, where we pass it `input` and `output`. A syntactic difference.

> Probably a result of the new 2.4.0 Keras release from 9 days ago: https://github.com/keras-team/keras/releases

I think, we can modify it to ``model = Model([input_a, input_b], similarity)`` so that it works for all versions.
@dsandeep0138 Are you able to make a PR?

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -314,7 +314,7 @@ def run(self):
     # See https://github.com/RaRe-Technologies/gensim/pull/2814#issuecomment-621477948
     linux_testenv += [
         'tensorflow',
-        'keras==2.3.1',
+        'keras',
     ]
 
 NUMPY_STR = 'numpy >= 1.11.3'


Test patch:
diff --git a/gensim/test/test_keras_integration.py b/gensim/test/test_keras_integration.py
--- a/gensim/test/test_keras_integration.py
+++ b/gensim/test/test_keras_integration.py
@@ -59,7 +59,7 @@ def testEmbeddingLayerCosineSim(self):
         embedding_b = embedding_layer(input_b)
         similarity = dot([embedding_a, embedding_b], axes=2, normalize=True)
 
-        model = Model(input=[input_a, input_b], output=similarity)
+        model = Model(inputs=[input_a, input_b], outputs=similarity)
         model.compile(optimizer='sgd', loss='mse')
 
         word_a = 'graph'