jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def __init__(self, model, data, config):
        self.model_train = model['train']
        self.model_eval = model['eval']
        self.data = data
        self.config = config

        self.loss_fn = BinaryCrossentropy()
        self.optimizer = Adam(learning_rate=self.config.learning_rate)
        self.metrics = ['accuracy', AUC()]
//  [python]
def define_metrics():
    '''Define the training metrics.'''
    return [
      BinaryAccuracy(name='accuracy'),
      F1_score,
      Precision(name='precision'),
      Recall(name='recall'),
      tf.keras.metrics.AUC(name='auc'),
    ]
//  [python]
def define_metrics():
    '''Define the training metrics.'''
    return [
      BinaryAccuracy(name='accuracy'),
      F1_score,
      Precision(name='precision'),
      Recall(name='recall'),
      AUC(name='auc'),
    ]
//  [python]
def create_woodbridge_rnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dga_woodbridge")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(LSTM(128))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", tf.keras.metrics.AUC()]
    )
    return model
//  [python]
def create_woodbridge_rnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dga_woodbridge")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(LSTM(128))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", AUC()]
    )
    return model
//  [python]
def create_model_fn():
  """Creates the model_fn to be used by the TPUEstimator."""

  def _dlrm_model_fn(features, mode, params):
    """Model function definition for DLRM.

    Args:
      features: List of feature tensors used in model.
      mode: Usage mode of the model function, e.g. train, eval, etc.
      params: Hparams for the model.

    Returns:
      TPUEstimatorSpec providing the train_op and loss operators.

    Raises:
      NotImplementedError for unsupported execution modes.
    """

    preds, host_call_fn = logits_fn(features, params, None)
    tf.logging.info("Model_FN: Shape of predictions: %s", preds.get_shape())
    labels = features["clicked"]
    tf.logging.info("Model_FN: Shape of labels: %s", labels.get_shape())

    if mode == tf.estimator.ModeKeys.EVAL:
      labels = tf.reshape(labels, [-1])
      preds = tf.reshape(preds, [-1])
      bce_func = tf.keras.losses.BinaryCrossentropy(
          from_logits=False, reduction=tf.compat.v2.keras.losses.Reduction.NONE)
      eval_loss = tf.reduce_mean(bce_func(labels, preds))

      def metric_fn(labels, predictions):
        label_weights = tf.ones_like(labels, dtype=tf.float32)
        prediction_labels = tf.round(predictions)

        return {
            utils.ACC_KEY:
                tf.metrics.accuracy(
                    labels=labels,
                    predictions=prediction_labels,
                    weights=label_weights),
            utils.AUC_KEY:
                tf.metrics.auc(
                    labels=labels,
                    predictions=predictions,
                    weights=label_weights,
                    num_thresholds=1000,
                    curve="ROC"),
        }

      eval_metrics = (metric_fn, [labels, preds])

      tf.logging.info(
          "Model_FN EVAL: Metrics have been set up. Now returning..")

      return tf.estimator.tpu.TPUEstimatorSpec(
          mode=mode,
          loss=eval_loss,
          host_call=host_call_fn,
          eval_metrics=eval_metrics)

    elif mode == tf.estimator.ModeKeys.TRAIN:

      bce_func = tf.keras.losses.BinaryCrossentropy(
          from_logits=False, reduction=tf.compat.v2.keras.losses.Reduction.NONE)
      loss = tf.reduce_mean(bce_func(labels, preds))

      global_step = tf.train.get_global_step()
      optimizer = tf.train.GradientDescentOptimizer(
          learning_rate=params["learning_rate"])
      optimizer = tf.tpu.CrossShardOptimizer(optimizer)
      train_op = optimizer.minimize(loss, global_step=global_step)

      return tf.estimator.tpu.TPUEstimatorSpec(
          mode=mode,
          loss=loss,
          train_op=train_op,
          host_call=host_call_fn,
      )

    else:
      raise NotImplementedError(
          "Only TRAIN and EVAL modes are supported. Got: %s" % (mode))

  return _dlrm_model_fn
//  [python]
def create_model_fn():
  """Creates the model_fn to be used by the TPUEstimator."""

  def _dlrm_model_fn(features, mode, params):
    """Model function definition for DLRM.

    Args:
      features: List of feature tensors used in model.
      mode: Usage mode of the model function, e.g. train, eval, etc.
      params: Hparams for the model.

    Returns:
      TPUEstimatorSpec providing the train_op and loss operators.

    Raises:
      NotImplementedError for unsupported execution modes.
    """

    preds, host_call_fn = logits_fn(features, params, None)
    tf.logging.info("Model_FN: Shape of predictions: %s", preds.get_shape())
    labels = features["clicked"]
    tf.logging.info("Model_FN: Shape of labels: %s", labels.get_shape())

    if mode == tf.estimator.ModeKeys.EVAL:
      labels = tf.reshape(labels, [-1])
      preds = tf.reshape(preds, [-1])
      bce_func = tf.keras.losses.BinaryCrossentropy(
          from_logits=False, reduction=tf.compat.v2.keras.losses.Reduction.NONE)
      eval_loss = tf.reduce_mean(bce_func(labels, preds))

      def metric_fn(labels, predictions):
        label_weights = tf.ones_like(labels, dtype=tf.float32)
        prediction_labels = tf.round(predictions)

        return {
            utils.ACC_KEY:
                tf.metrics.accuracy(
                    labels=labels,
                    predictions=prediction_labels,
                    weights=label_weights),
            utils.AUC_KEY:
                tf.keras.metrics.AUC(
                    num_thresholds=1000,
                    curve="ROC")(labels, predictions),
        }

      eval_metrics = (metric_fn, [labels, preds])

      tf.logging.info(
          "Model_FN EVAL: Metrics have been set up. Now returning..")

      return tf.estimator.tpu.TPUEstimatorSpec(
          mode=mode,
          loss=eval_loss,
          host_call=host_call_fn,
          eval_metrics=eval_metrics)

    elif mode == tf.estimator.ModeKeys.TRAIN:

      bce_func = tf.keras.losses.BinaryCrossentropy(
          from_logits=False, reduction=tf.compat.v2.keras.losses.Reduction.NONE)
      loss = tf.reduce_mean(bce_func(labels, preds))

      global_step = tf.train.get_global_step()
      optimizer = tf.train.GradientDescentOptimizer(
          learning_rate=params["learning_rate"])
      optimizer = tf.tpu.CrossShardOptimizer(optimizer)
      train_op = optimizer.minimize(loss, global_step=global_step)

      return tf.estimator.tpu.TPUEstimatorSpec(
          mode=mode,
          loss=loss,
          train_op=train_op,
          host_call=host_call_fn,
      )

    else:
      raise NotImplementedError(
          "Only TRAIN and EVAL modes are supported. Got: %s" % (mode))

  return _dlrm_model_fn
//  [python]
def create_rnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dns_rnn")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(LSTM(256, return_sequences=True))
    model.add(LSTM(128))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", tf.keras.metrics.AUC()]
    )
    return model
//  [python]
def create_rnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dns_rnn")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(LSTM(256, return_sequences=True))
    model.add(LSTM(128))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", AUC()]
    )
    return model
//  [python]
def create_vosoughi_rcnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dns_rcnn")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(Conv1D(filters=128, kernel_size=8, activation="relu"))
    model.add(GlobalMaxPooling1D())
    model.add(Reshape((1, 128)))
    model.add(LSTM(128))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", tf.keras.metrics.AUC()]
    )
    return model
//  [python]
def create_vosoughi_rcnn(num_classes: int = 2) -> tf.keras.Model:
    model = Sequential(name="dns_rcnn")
    model.add(Embedding(en2vec.corpus_size + 1, 128, input_length=256))
    model.add(Conv1D(filters=128, kernel_size=8, activation="relu"))
    model.add(GlobalMaxPooling1D())
    model.add(Reshape((1, 128)))
    model.add(LSTM(128))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(1024, activation="selu", kernel_initializer="lecun_normal"))
    model.add(AlphaDropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", AUC()]
    )
    return model
//  [python]
def create_ffnn(num_classes: int = 2):
    model = Sequential(name="dns_ffw")
    model.add(Dense(128, activation="relu", input_dim=18))
    model.add(Dropout(0.1))
    model.add(Dense(128, activation="relu"))
    model.add(Dense(128, activation="tanh"))
    model.add(Dropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        loss="categorical_crossentropy",
        optimizer="nadam",
        metrics=["accuracy", tf.keras.metrics.AUC()],
    )
    return model
//  [python]
def create_ffnn(num_classes: int = 2):
    model = Sequential(name="dns_ffw")
    model.add(Dense(128, activation="relu", input_dim=18))
    model.add(Dropout(0.1))
    model.add(Dense(128, activation="relu"))
    model.add(Dense(128, activation="tanh"))
    model.add(Dropout(0.1))
    model.add(Dense(num_classes, activation="softmax"))

    model.compile(
        loss="categorical_crossentropy",
        optimizer="nadam",
        metrics=["accuracy", AUC()],
    )
    return model
//  [python]
def create_cnn(num_classes: int = 2) -> tf.keras.Model:
    x = Input(shape=(256,), dtype="int64")
    h = Embedding(en2vec.corpus_size + 1, 128, input_length=256)(x)

    conv1 = Convolution1D(filters=256, kernel_size=10, activation="tanh")(h)
    conv2 = Convolution1D(filters=256, kernel_size=7, activation="tanh")(h)
    conv3 = Convolution1D(filters=256, kernel_size=5, activation="tanh")(h)
    conv4 = Convolution1D(filters=256, kernel_size=3, activation="tanh")(h)

    h = Concatenate()(
        [
            GlobalMaxPooling1D()(conv1),
            GlobalMaxPooling1D()(conv2),
            GlobalMaxPooling1D()(conv3),
            GlobalMaxPooling1D()(conv4),
        ]
    )

    h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
    h = AlphaDropout(0.1)(h)
    h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
    h = AlphaDropout(0.1)(h)

    y = Dense(num_classes, activation="softmax")(h)

    model = Model(x, y)
    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", tf.keras.metrics.AUC()]
    )
    return model
//  [python]
def create_cnn(num_classes: int = 2) -> tf.keras.Model:
    x = Input(shape=(256,), dtype="int64")
    h = Embedding(en2vec.corpus_size + 1, 128, input_length=256)(x)

    conv1 = Convolution1D(filters=256, kernel_size=10, activation="tanh")(h)
    conv2 = Convolution1D(filters=256, kernel_size=7, activation="tanh")(h)
    conv3 = Convolution1D(filters=256, kernel_size=5, activation="tanh")(h)
    conv4 = Convolution1D(filters=256, kernel_size=3, activation="tanh")(h)

    h = Concatenate()(
        [
            GlobalMaxPooling1D()(conv1),
            GlobalMaxPooling1D()(conv2),
            GlobalMaxPooling1D()(conv3),
            GlobalMaxPooling1D()(conv4),
        ]
    )

    h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
    h = AlphaDropout(0.1)(h)
    h = Dense(1024, activation="selu", kernel_initializer="lecun_normal")(h)
    h = AlphaDropout(0.1)(h)

    y = Dense(num_classes, activation="softmax")(h)

    model = Model(x, y)
    model.compile(
        optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy", AUC()]
    )
    return model
//  [python]
def __init__(self, model, data, config, exp_name_time,sweep_string=None):
        self.model_train = model['train']
        # self.model_eval = model['eval']
        self.data = data
        self.config = config
        self.exp_name_time = exp_name_time
        self.sweep_string = sweep_string
        # self.loss_fn = BinaryCrossentropy()
        # self.loss_fn = auc_max

        if config.learn_background:
            self.loss_fn = 'categorical_crossentropy'
        elif config.loss == "AUC":
            self.loss_fn = AUCLoss()
        elif config.loss == "BinaryCrossentropy":
            self.loss_fn = BinaryCrossentropy()
        else:
            self.loss_fn = None

        self.optimizer = Adam(learning_rate=self.config.learning_rate)
        self.metrics = ['accuracy', tf.keras.metrics.AUC()]
//  [python]
def __init__(self, model, data, config, exp_name_time,sweep_string=None):
        self.model_train = model['train']
        # self.model_eval = model['eval']
        self.data = data
        self.config = config
        self.exp_name_time = exp_name_time
        self.sweep_string = sweep_string
        # self.loss_fn = BinaryCrossentropy()
        # self.loss_fn = auc_max

        if config.learn_background:
            self.loss_fn = 'categorical_crossentropy'
        elif config.loss == "AUC":
            self.loss_fn = AUCLoss()
        elif config.loss == "BinaryCrossentropy":
            self.loss_fn = BinaryCrossentropy()
        else:
            self.loss_fn = None

        self.optimizer = Adam(learning_rate=self.config.learning_rate)
        self.metrics = ['accuracy', AUC()]
//  [python]
def build_model(self, num_classes=2):
        input_shape = (self.img_size[0], self.img_size[1], 3)
        self.model = self.build_complex_model(input_shape, num_classes)

        self.model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
                           loss='binary_crossentropy',
                           metrics=[
                               Recall(name='recall'),
                               tf.metrics.AUC(name='auc'),
                               Precision(name='precision'),
                               'accuracy',
                               tf.keras.metrics.BinaryAccuracy(name='binary_accuracy'),
                           ])
        print("Model Summary:")
        self.model.summary()
//  [python]
def build_model(self, num_classes=2):
        input_shape = (self.img_size[0], self.img_size[1], 3)
        self.model = self.build_complex_model(input_shape, num_classes)

        self.model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
                           loss='binary_crossentropy',
                           metrics=[
                               Recall(name='recall'),
                               tf.keras.metrics.AUC(name='auc'),
                               Precision(name='precision'),
                               'accuracy',
                               tf.keras.metrics.BinaryAccuracy(name='binary_accuracy'),
                           ])
        print("Model Summary:")
        self.model.summary()
//  [python]
def build(self):
        model = models.Sequential()
        model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(64, (3, 3), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(128, (3, 3), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Flatten())
        model.add(layers.Dropout(0.5))  # Dropout for regularization
        model.add(layers.Dense(512, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(256, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=Adam(self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def build(self):
        model = models.Sequential()
        model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(64, (3, 3), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(128, (3, 3), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Flatten())
        model.add(layers.Dropout(0.5))  # Dropout for regularization
        model.add(layers.Dense(512, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(256, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer='adam',
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def build_tuning(self, hp):

        model = models.Sequential()
        model.add(layers.Conv2D(hp.Int('filters_1', 16, 128, step=16), (3, 3), activation='relu',
                                input_shape=(self.input_width_height, self.input_width_height, self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        for i in range(hp.Int('conv_blocks', 1, 2, default=1)):
            model.add(layers.Conv2D(hp.Int('filters_' + str(i), 32, 64, step=32), (3, 3), activation='relu'))
            model.add(layers.MaxPooling2D((2, 2)))
            #if hp.Choice('pooling_' + str(i), ['avg', 'max']) == 'max':
            #    x = tf.keras.layers.MaxPool2D()(x)
            #else:
            #    x = tf.keras.layers.AvgPool2D()(x)
        model.add(layers.Flatten())
        model.add(layers.Dense(hp.Int('hidden_size', 256, 512, step=128, default=512), activation='relu'))
        model.add(layers.Dropout(hp.Float('dropout', 0.3, 0.5, step=0.1, default=0.5)))
        model.add(layers.Dense(self.num_classes, activation='softmax'))
        # activation=hp.Choice('act_1', ['relu', 'tanh'])

        model.compile(loss='categorical_crossentropy', optimizer='adam',
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    embed_dim = 8
    mode = 'in'
    dnn_dropout = 0.5
    hidden_units = [256, 128, 64]

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10
    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                                         embed_dim=embed_dim,
                                                         read_part=read_part,
                                                         sample_num=sample_num,
                                                         test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = PNN(feature_columns, hidden_units, dnn_dropout)
    model.summary()
    # ============================model checkpoint======================
    # check_path = 'save/pnn_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # =========================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[tf.keras.metrics.AUC()])
    # ===========================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        callbacks=[EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True)],  # checkpoint
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    embed_dim = 8
    mode = 'in'
    dnn_dropout = 0.5
    hidden_units = [256, 128, 64]

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10
    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                                         embed_dim=embed_dim,
                                                         read_part=read_part,
                                                         sample_num=sample_num,
                                                         test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = PNN(feature_columns, hidden_units, dnn_dropout)
    model.summary()
    # ============================model checkpoint======================
    # check_path = 'save/pnn_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # =========================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[AUC()])
    # ===========================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        callbacks=[EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True)],  # checkpoint
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
def __init__(self, maxlen=128, batch_size=32, w_embed_size=200, padding="post", h_embed_size=200, dropout=0.1, patience=1, plot=True, max_epochs=100):
        self.maxlen = maxlen
        self.METRICS = [BinaryAccuracy(name='accuracy'),
                        Precision(name='precision'),
                        Recall(name='recall'),
                        tf.keras.metrics.AUC(name='auc')]
        self.w_embed_size = w_embed_size
        self.h_embed_size = h_embed_size
        self.dropout = dropout
        self.vocab_size = -1
        self.padding = padding
        self.patience = patience
        self.model = None
        self.w2i = {}
        self.epochs = max_epochs
        self.i2w = {}
        self.vocab = []
        self.batch_size = batch_size
        self.show_the_model = plot
        self.threshold = 0.2
        self.toxic_label = 2
        self.not_toxic_label = 1
        self.unk_token = "[unk]"
        self.pad_token = "[pad]"
//  [python]
def __init__(self, maxlen=128, batch_size=32, w_embed_size=200, padding="post", h_embed_size=200, dropout=0.1, patience=1, plot=True, max_epochs=100):
        self.maxlen = maxlen
        self.METRICS = [BinaryAccuracy(name='accuracy'),
                        Precision(name='precision'),
                        Recall(name='recall'),
                        AUC(name='auc')]
        self.w_embed_size = w_embed_size
        self.h_embed_size = h_embed_size
        self.dropout = dropout
        self.vocab_size = -1
        self.padding = padding
        self.patience = patience
        self.model = None
        self.w2i = {}
        self.epochs = max_epochs
        self.i2w = {}
        self.vocab = []
        self.batch_size = batch_size
        self.show_the_model = plot
        self.threshold = 0.2
        self.toxic_label = 2
        self.not_toxic_label = 1
        self.unk_token = "[unk]"
        self.pad_token = "[pad]"
//  [python]
def create_model(X, outputdim, learning_rate, regularization_rate,
                 filters, layers, kernel_size, fc_nodes, dropout_rate):
    weightinit = 'lecun_uniform'  # weight initialization

    model = Sequential()

    model.add(BatchNormalization(input_shape=(X.shape[1], X.shape[2])))

    filters_list = [filters] * layers

    for filter_number in filters_list:
        model.add(
            Convolution1D(filter_number,
                          kernel_size=(kernel_size,),
                          padding='same',
                          kernel_regularizer=l2(regularization_rate),
                          kernel_initializer=weightinit))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Dropout(rate=dropout_rate))

    model.add(Flatten())

    model.add(
        Dense(units=fc_nodes,
              kernel_regularizer=l2(regularization_rate),
              kernel_initializer=weightinit))  # Fully connected layer
    model.add(Activation('relu'))  # Relu activation
    model.add(Dropout(rate=dropout_rate))

    model.add(Dense(units=outputdim, kernel_initializer=weightinit))
    model.add(BatchNormalization())
    model.add(Activation("softmax"))  # Final classification layer

    model.compile(loss='categorical_crossentropy',
                  optimizer=Adam(learning_rate=learning_rate),
                  metrics=['AUC'])

    return model
//  [python]
def create_model(X, outputdim, learning_rate, regularization_rate,
                 filters, layers, kernel_size, fc_nodes, dropout_rate):
    weightinit = 'lecun_uniform'  # weight initialization

    model = Sequential()

    model.add(BatchNormalization(input_shape=(X.shape[1], X.shape[2])))

    filters_list = [filters] * layers

    for filter_number in filters_list:
        model.add(
            Convolution1D(filter_number,
                          kernel_size=(kernel_size,),
                          padding='same',
                          kernel_regularizer=l2(regularization_rate),
                          kernel_initializer=weightinit))
        model.add(BatchNormalization())
        model.add(Activation('relu'))
        model.add(Dropout(rate=dropout_rate))

    model.add(Flatten())

    model.add(
        Dense(units=fc_nodes,
              kernel_regularizer=l2(regularization_rate),
              kernel_initializer=weightinit))  # Fully connected layer
    model.add(Activation('relu'))  # Relu activation
    model.add(Dropout(rate=dropout_rate))

    model.add(Dense(units=outputdim, kernel_initializer=weightinit))
    model.add(BatchNormalization())
    model.add(Activation("softmax"))  # Final classification layer

    model.compile(loss='categorical_crossentropy',
                  optimizer=Adam(learning_rate=learning_rate),
                  metrics=[AUC(name='auc', curve='PR')])

    return model
//  [python]
def build(self):
        model = models.Sequential()
        model.add(QConv(filter_size=2, depth=8, activation='relu', name='qconv1',
                        input_shape=(self.input_width_height, self.input_width_height, self.channels)))
        model.add(layers.Conv2D(16, (2, 2), activation='relu'))
        model.add(layers.Flatten())
        model.add(layers.Dense(32, activation='relu'))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=Adam(self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def build(self):
        model = models.Sequential()
        model.add(layers.Conv2D(32, (2, 2), activation='relu', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(32, (2, 2), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Flatten())
        model.add(layers.Dense(20, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), tf.keras.metrics.AUC(name='auc')])

        return model
//  [python]
def build(self):
        model = models.Sequential()
        model.add(layers.Conv2D(32, (2, 2), activation='relu', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(32, (2, 2), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Flatten())
        model.add(layers.Dense(20, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def run(train_data, test_data, feature_columns, args):
    # 1. 建模
    model = DeepFM(feature_columns=feature_columns,
                   dnn_hidden_units=args.hidden_units,
                   dnn_drop_rate=args.drop_rate,
                   dnn_use_bn=args.use_bn)
    # 2. 编译
    model.compile(optimizer=Adam(learning_rate=args.learning_rate),
                  loss="binary_crossentropy",
                  metrics=['AUC'])
    model.summary()
    # 3. 训练
    model.fit(train_data[0],
              train_data[1],
              batch_size=args.batch_size,
              epochs=args.epochs,
              callbacks=[EarlyStopping(monitor='val_loss',
                                       patience=args.patience,
                                       mode='min',
                                       restore_best_weights=args.restore_best_weights)],
              validation_split=args.val_splite,
              )
    # 4. 测试
    print('test AUC: %f' % model.evaluate(test_data[0], test_data[1], batch_size=args.batch_size)[1])
//  [python]
def run(train_data, test_data, feature_columns, args):
    # 1. 建模
    model = DeepFM(feature_columns=feature_columns,
                   dnn_hidden_units=args.hidden_units,
                   dnn_drop_rate=args.drop_rate,
                   dnn_use_bn=args.use_bn)
    # 2. 编译
    model.compile(optimizer=Adam(learning_rate=args.learning_rate),
                  loss="binary_crossentropy",
                  metrics=[AUC()])
    model.summary()
    # 3. 训练
    model.fit(train_data[0],
              train_data[1],
              batch_size=args.batch_size,
              epochs=args.epochs,
              callbacks=[EarlyStopping(monitor='val_loss',
                                       patience=args.patience,
                                       mode='min',
                                       restore_best_weights=args.restore_best_weights)],
              validation_split=args.val_splite,
              )
    # 4. 测试
    print('test AUC: %f' % model.evaluate(test_data[0], test_data[1], batch_size=args.batch_size)[1])
//  [python]
batch_size = 128
epochs = 10
opt = Adam(learning_rate=0.003, decay=0.0001)
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
    mmoe = MMOE(cate_feature_columns=cate_feature_dict, cate_feature_dict=[user_cate_feature_dict, item_cate_feature_dict])
    model = mmoe.build_graph()
    model.summary()
    # ============================Compile============================
    model.compile(loss=["binary_crossentropy", "binary_crossentropy"],
                  loss_weights=[1.0, 1.0],
                  optimizer=opt,
                  metrics=[tf.keras.metrics.AUC()])
    model.run_eagerly = True
//  [python]
batch_size = 128
epochs = 10
opt = Adam(learning_rate=0.003, decay=0.0001)
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
    mmoe = MMOE(cate_feature_columns=cate_feature_dict, cate_feature_dict=[user_cate_feature_dict, item_cate_feature_dict])
    model = mmoe.build_graph()
    model.summary()
    # ============================Compile============================
    model.compile(loss=["binary_crossentropy", "binary_crossentropy"],
                  loss_weights=[1.0, 1.0],
                  optimizer=opt,
                  metrics=[AUC()])
    model.run_eagerly = True
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    # you can modify your file path
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    embed_dim = 8
    dnn_dropout = 0.5
    hidden_units = [256, 128, 64]

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10

    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                                         embed_dim=embed_dim,
                                                         read_part=read_part,
                                                         sample_num=sample_num,
                                                         test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = Deep_Crossing(feature_columns, hidden_units)
    model.summary()
    # ============================model checkpoint======================
    # check_path = 'save/deep_crossing_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # =========================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[tf.keras.metrics.AUC()])
    # ===========================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        callbacks=[EarlyStopping(monitor='val_auc', patience=2, restore_best_weights=True)],  # checkpoint
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    # you can modify your file path
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    embed_dim = 8
    dnn_dropout = 0.5
    hidden_units = [256, 128, 64]

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10

    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                                         embed_dim=embed_dim,
                                                         read_part=read_part,
                                                         sample_num=sample_num,
                                                         test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = Deep_Crossing(feature_columns, hidden_units)
    model.summary()
    # ============================model checkpoint======================
    # check_path = 'save/deep_crossing_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # =========================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[AUC()])
    # ===========================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        callbacks=[EarlyStopping(monitor='val_auc', patience=2, restore_best_weights=True)],  # checkpoint
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    # you can modify your file path
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    k = 10

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10

    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                           read_part=read_part,
                                           sample_num=sample_num,
                                           test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = FM(feature_columns=feature_columns, k=k)
    model.summary()
    # ============================model checkpoint======================
    # check_path = '../save/fm_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # ============================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=['AUC'])
    # ==============================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        # callbacks=[checkpoint],
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'


if __name__ == '__main__':
    # ========================= Hyper Parameters =======================
    # you can modify your file path
    file = '../dataset/Criteo/train.txt'
    read_part = True
    sample_num = 5000000
    test_size = 0.2

    k = 10

    learning_rate = 0.001
    batch_size = 4096
    epochs = 10

    # ========================== Create dataset =======================
    feature_columns, train, test = create_criteo_dataset(file=file,
                                           read_part=read_part,
                                           sample_num=sample_num,
                                           test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # ============================Build Model==========================
    model = FM(feature_columns=feature_columns, k=k)
    model.summary()
    # ============================model checkpoint======================
    # check_path = '../save/fm_weights.epoch_{epoch:04d}.val_loss_{val_loss:.4f}.ckpt'
    # checkpoint = tf.keras.callbacks.ModelCheckpoint(check_path, save_weights_only=True,
    #                                                 verbose=1, period=5)
    # ============================Compile============================
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[AUC()])
    # ==============================Fit==============================
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        # callbacks=[checkpoint],
        batch_size=batch_size,
        validation_split=0.1
    )
    # ===========================Test==============================
    print('test AUC: %f' % model.evaluate(test_X, test_y)[1])
//  [python]
def __init__(self, model, data, config):
        self.model_train = model['train']
        # self.model_eval = model['eval']
        self.data = data
        self.config = config

        # self.loss_fn = BinaryCrossentropy()
        # self.loss_fn = auc_max

        if config.learn_background:
            self.loss_fn = 'categorical_crossentropy'
        elif config.loss == "AUC":
            self.loss_fn = AUCLoss()
        elif config.loss == "BinaryCrossentropy":
            self.loss_fn = BinaryCrossentropy()
        else:
            self.loss_fn = None

        self.optimizer = Adam(learning_rate=self.config.learning_rate)
        self.metrics = ['accuracy', tf.keras.metrics.AUC()]
//  [python]
def __init__(self, model, data, config):
        self.model_train = model['train']
        # self.model_eval = model['eval']
        self.data = data
        self.config = config

        # self.loss_fn = BinaryCrossentropy()
        # self.loss_fn = auc_max

        if config.learn_background:
            self.loss_fn = 'categorical_crossentropy'
        elif config.loss == "AUC":
            self.loss_fn = AUCLoss()
        elif config.loss == "BinaryCrossentropy":
            self.loss_fn = BinaryCrossentropy()
        else:
            self.loss_fn = None

        self.optimizer = Adam(learning_rate=self.config.learning_rate)
        self.metrics = ['accuracy', AUC()]
//  [python]
def build(self):

        model = models.Sequential()
        model.add(layers.Conv2D(32, (5, 5), activation='relu', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(64, (5, 5), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(128, (5, 5), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(256, (5, 5), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Conv2D(256, (5, 5), activation='relu'))
        model.add(layers.MaxPooling2D((2, 2)))
        model.add(layers.Flatten())
        model.add(layers.Dense(2048, activation='relu'))
        model.add(layers.Dropout(0.5))
        model.add(layers.Dense(256, activation='relu'))
        model.add(layers.Dense(64, activation='relu'))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=SGD(learning_rate=self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def build(self):

        model = models.Sequential()
        model.add(layers.Conv2D(6, 5, activation='tanh', input_shape=(self.input_width_height,
                                                                            self.input_width_height,
                                                                            self.channels)))
        model.add(layers.AveragePooling2D(2))
        model.add(layers.Activation('sigmoid'))
        model.add(layers.Conv2D(16, 5, activation='tanh'))
        model.add(layers.AveragePooling2D(2))
        model.add(layers.Activation('sigmoid'))
        model.add(layers.Conv2D(120, 5, activation='tanh'))
        model.add(layers.Flatten())
        model.add(layers.Dense(84, activation='tanh'))
        model.add(layers.Dense(self.num_classes, activation='softmax'))

        model.compile(loss='categorical_crossentropy', optimizer=Adam(self.learning_rate),
                      metrics=['acc', Precision(name="prec"), Recall(name="rec"), AUC(name='auc')])

        return model
//  [python]
def easy_demo(file, sample_num=500000, read_part=True, test_size=0.1, epochs=10):
    feature_columns, train, test = create_small_criteo_dataset(file=file,
                                                         read_part=read_part,
                                                         sample_num=sample_num,
                                                         test_size=test_size)
    train_X, train_y = train
    test_X, test_y = test
    # TODO: Build Model
    model = FM(feature_columns=feature_columns, **model_params)
    model.summary()
    model.compile(loss=binary_crossentropy, optimizer=Adam(learning_rate=learning_rate),
                  metrics=[AUC()])
    # TODO: Fit
    model.fit(
        train_X,
        train_y,
        epochs=epochs,
        batch_size=batch_size,
        validation_split=0.1
    )
    # TODO: Test
    print('test AUC: %f' % model.evaluate(test_X, test_y, batch_size=batch_size)[1])
//  [python]
def evaluate_legacy(model, weights, test_gen, loss, optimiser, loss_weights):
    # Get loss weights
    metrics = ['accuracy', tf.keras.metrics.AUC(curve='ROC', name='auroc'), tf.keras.metrics.AUC(curve='PR', name='aupr')]

    # Compile model
    model.compile(loss=loss, optimizer=optimiser, metrics=metrics, loss_weights=loss_weights)

    # Evaluate
    print(f'Evaluating weights file: {weights}')
    output = model.evaluate(test_gen.dataset)

    # Report results
    loss = output[0]
    audio_acc = output[4]
    video_acc = output[7]
    av_acc = output[10]
    auroc = output[11]
    aupr = output[12]
    print_results(weights, loss, audio_acc, video_acc, av_acc, auroc, aupr)