node {
  name: "global_step/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "global_step"
  op: "VarHandleOp"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "shape"
    value {
      shape {
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: "global_step"
    }
  }
}
node {
  name: "global_step/IsInitialized/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/Assign"
  op: "AssignVariableOp"
  input: "global_step"
  input: "global_step/Initializer/zeros"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "global_step"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/VarIsInitializedOp"
  op: "VarIsInitializedOp"
  input: "global_step"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Switch"
  op: "Switch"
  input: "global_step/VarIsInitializedOp"
  input: "global_step/VarIsInitializedOp"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/switch_t"
  op: "Identity"
  input: "global_step/cond/Switch:1"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/switch_f"
  op: "Identity"
  input: "global_step/cond/Switch"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/pred_id"
  op: "Identity"
  input: "global_step/VarIsInitializedOp"
  attr {
    key: "T"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Read/ReadVariableOp"
  op: "ReadVariableOp"
  input: "global_step/cond/Read/ReadVariableOp/Switch:1"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
}
node {
  name: "global_step/cond/Read/ReadVariableOp/Switch"
  op: "Switch"
  input: "global_step"
  input: "global_step/cond/pred_id"
  attr {
    key: "T"
    value {
      type: DT_RESOURCE
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Identity"
  op: "Identity"
  input: "global_step/cond/Read/ReadVariableOp"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Switch_1"
  op: "Switch"
  input: "global_step/Initializer/zeros"
  input: "global_step/cond/pred_id"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@global_step"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/cond/Merge"
  op: "Merge"
  input: "global_step/cond/Switch_1"
  input: "global_step/cond/Identity"
  attr {
    key: "N"
    value {
      i: 2
    }
  }
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
        shape {
        }
      }
    }
  }
}
node {
  name: "global_step/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "global_step/add"
  op: "Add"
  input: "global_step/cond/Merge"
  input: "global_step/add/y"
  attr {
    key: "T"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "Const"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 15
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
          dim {
            size: 15
          }
        }
        string_val: "../data/tf_records/discharge_summary.tfrecord"
        string_val: "../data/tf_records/physician.tfrecord"
        string_val: "../data/tf_records/nursing.tfrecord"
        string_val: "../data/tf_records/nursing_other.tfrecord"
        string_val: "../data/tf_records/radiology.tfrecord"
        string_val: "../data/tf_records/general.tfrecord"
        string_val: "../data/tf_records/respiratory.tfrecord"
        string_val: "../data/tf_records/consult.tfrecord"
        string_val: "../data/tf_records/nutrition.tfrecord"
        string_val: "../data/tf_records/case_management.tfrecord"
        string_val: "../data/tf_records/pharmacy.tfrecord"
        string_val: "../data/tf_records/rehab_services.tfrecord"
        string_val: "../data/tf_records/social_work.tfrecord"
        string_val: "../data/tf_records/ecg.tfrecord"
        string_val: "../data/tf_records/echo.tfrecord"
      }
    }
  }
}
node {
  name: "count"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: -1
      }
    }
  }
}
node {
  name: "buffer_size"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 15
      }
    }
  }
}
node {
  name: "seed"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "seed2"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "cycle_length"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 4
      }
    }
  }
}
node {
  name: "block_length"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 1
      }
    }
  }
}
node {
  name: "sloppy"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_BOOL
        tensor_shape {
        }
        bool_val: true
      }
    }
  }
}
node {
  name: "buffer_output_elements"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 2
      }
    }
  }
}
node {
  name: "prefetch_input_elements"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 8
      }
    }
  }
}
node {
  name: "buffer_size_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 100
      }
    }
  }
}
node {
  name: "seed_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "seed2_1"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 0
      }
    }
  }
}
node {
  name: "batch_size"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 32
      }
    }
  }
}
node {
  name: "num_parallel_calls"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT64
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT64
        tensor_shape {
        }
        int64_val: 128
      }
    }
  }
}
node {
  name: "drop_remainder"
  op: "Const"
  device: "/device:CPU:0"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_BOOL
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_BOOL
        tensor_shape {
        }
        bool_val: true
      }
    }
  }
}
node {
  name: "IteratorV2"
  op: "IteratorV2"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_FLOAT
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "TensorSliceDataset"
  op: "TensorSliceDataset"
  input: "Const"
  attr {
    key: "Toutput_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "RepeatDataset"
  op: "RepeatDataset"
  input: "TensorSliceDataset"
  input: "count"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
}
node {
  name: "ShuffleDataset"
  op: "ShuffleDataset"
  input: "RepeatDataset"
  input: "buffer_size"
  input: "seed"
  input: "seed2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "reshuffle_each_iteration"
    value {
      b: true
    }
  }
}
node {
  name: "ParallelInterleaveDataset"
  op: "ParallelInterleaveDataset"
  input: "ShuffleDataset"
  input: "cycle_length"
  input: "block_length"
  input: "sloppy"
  input: "buffer_output_elements"
  input: "prefetch_input_elements"
  attr {
    key: "Targuments"
    value {
      list {
      }
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "f"
    value {
      func {
        name: "tf_data_structured_function_wrapper_rn3e6kArW78"
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
}
node {
  name: "ShuffleDataset_1"
  op: "ShuffleDataset"
  input: "ParallelInterleaveDataset"
  input: "buffer_size_1"
  input: "seed_1"
  input: "seed2_1"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_STRING
      }
    }
  }
  attr {
    key: "reshuffle_each_iteration"
    value {
      b: true
    }
  }
}
node {
  name: "MapAndBatchDatasetV2"
  op: "MapAndBatchDatasetV2"
  input: "ShuffleDataset_1"
  input: "batch_size"
  input: "num_parallel_calls"
  input: "drop_remainder"
  attr {
    key: "Targuments"
    value {
      list {
      }
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "f"
    value {
      func {
        name: "tf_data_structured_function_wrapper_9z4XDFOWUdQ"
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_FLOAT
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "MakeIterator"
  op: "MakeIterator"
  input: "MapAndBatchDatasetV2"
  input: "IteratorV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@IteratorV2"
      }
    }
  }
}
node {
  name: "IteratorToStringHandle"
  op: "IteratorToStringHandle"
  input: "IteratorV2"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "IteratorGetNext"
  op: "IteratorGetNext"
  input: "IteratorV2"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 20
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
        }
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "output_types"
    value {
      list {
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_INT32
        type: DT_FLOAT
        type: DT_INT32
        type: DT_INT32
      }
    }
  }
}
node {
  name: "bert/embeddings/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "bert/embeddings/ExpandDims"
  op: "ExpandDims"
  input: "IteratorGetNext"
  input: "bert/embeddings/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "Dq\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Initializer/truncated_normal"
  op: "Add"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 28996
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/Assign"
  op: "Assign"
  input: "bert/embeddings/word_embeddings"
  input: "bert/embeddings/word_embeddings/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/word_embeddings/read"
  op: "Identity"
  input: "bert/embeddings/word_embeddings"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 28996
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/embedding_lookup/axis"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 0
      }
    }
  }
}
node {
  name: "bert/embeddings/embedding_lookup"
  op: "GatherV2"
  input: "bert/embeddings/word_embeddings/read"
  input: "bert/embeddings/ExpandDims"
  input: "bert/embeddings/embedding_lookup/axis"
  attr {
    key: "Taxis"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tindices"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tparams"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/word_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/embedding_lookup/Identity"
  op: "Identity"
  input: "bert/embeddings/embedding_lookup"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape"
  op: "Reshape"
  input: "bert/embeddings/embedding_lookup/Identity"
  input: "bert/embeddings/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\002\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal"
  op: "Add"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 2
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/Assign"
  op: "Assign"
  input: "bert/embeddings/token_type_embeddings"
  input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/token_type_embeddings/read"
  op: "Identity"
  input: "bert/embeddings/token_type_embeddings"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/token_type_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: -1
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape_1"
  op: "Reshape"
  input: "IteratorGetNext:6"
  input: "bert/embeddings/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/one_hot/on_value"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/embeddings/one_hot/off_value"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/one_hot/depth"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "bert/embeddings/one_hot"
  op: "OneHot"
  input: "bert/embeddings/Reshape_1"
  input: "bert/embeddings/one_hot/depth"
  input: "bert/embeddings/one_hot/on_value"
  input: "bert/embeddings/one_hot/off_value"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "TI"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "axis"
    value {
      i: -1
    }
  }
}
node {
  name: "bert/embeddings/MatMul"
  op: "MatMul"
  input: "bert/embeddings/one_hot"
  input: "bert/embeddings/token_type_embeddings/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/embeddings/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape_2"
  op: "Reshape"
  input: "bert/embeddings/MatMul"
  input: "bert/embeddings/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/add"
  op: "Add"
  input: "bert/embeddings/Reshape"
  input: "bert/embeddings/Reshape_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 128
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
        }
        int_val: 512
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/LessEqual"
  op: "LessEqual"
  input: "bert/embeddings/assert_less_equal/x"
  input: "bert/embeddings/assert_less_equal/y"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Const"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/All"
  op: "All"
  input: "bert/embeddings/assert_less_equal/LessEqual"
  input: "bert/embeddings/assert_less_equal/Const"
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: false
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Const"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: ""
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Const_1"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:x (bert/embeddings/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Const_2"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (bert/embeddings/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Assert/data_0"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: ""
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Assert/data_1"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "Condition x <= y did not hold element-wise:x (bert/embeddings/assert_less_equal/x:0) = "
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Assert/data_3"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_STRING
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_STRING
        tensor_shape {
        }
        string_val: "y (bert/embeddings/assert_less_equal/y:0) = "
      }
    }
  }
}
node {
  name: "bert/embeddings/assert_less_equal/Assert/Assert"
  op: "Assert"
  input: "bert/embeddings/assert_less_equal/All"
  input: "bert/embeddings/assert_less_equal/Assert/Assert/data_0"
  input: "bert/embeddings/assert_less_equal/Assert/Assert/data_1"
  input: "bert/embeddings/assert_less_equal/x"
  input: "bert/embeddings/assert_less_equal/Assert/Assert/data_3"
  input: "bert/embeddings/assert_less_equal/y"
  attr {
    key: "T"
    value {
      list {
        type: DT_STRING
        type: DT_STRING
        type: DT_INT32
        type: DT_STRING
        type: DT_INT32
      }
    }
  }
  attr {
    key: "summarize"
    value {
      i: 3
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\002\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Initializer/truncated_normal"
  op: "Add"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 512
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/Assign"
  op: "Assign"
  input: "bert/embeddings/position_embeddings"
  input: "bert/embeddings/position_embeddings/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/position_embeddings/read"
  op: "Identity"
  input: "bert/embeddings/position_embeddings"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/position_embeddings"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 512
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/Slice/begin"
  op: "Const"
  input: "^bert/embeddings/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\000\000\000\000\000\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/Slice/size"
  op: "Const"
  input: "^bert/embeddings/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\200\000\000\000\377\377\377\377"
      }
    }
  }
}
node {
  name: "bert/embeddings/Slice"
  op: "Slice"
  input: "bert/embeddings/position_embeddings/read"
  input: "bert/embeddings/Slice/begin"
  input: "bert/embeddings/Slice/size"
  attr {
    key: "Index"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape_3/shape"
  op: "Const"
  input: "^bert/embeddings/assert_less_equal/Assert/Assert"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: "\001\000\000\000\200\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/Reshape_3"
  op: "Reshape"
  input: "bert/embeddings/Slice"
  input: "bert/embeddings/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/add_1"
  op: "Add"
  input: "bert/embeddings/add"
  input: "bert/embeddings/Reshape_3"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/embeddings/LayerNorm/beta"
  input: "bert/embeddings/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/embeddings/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/embeddings/LayerNorm/gamma"
  input: "bert/embeddings/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/embeddings/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/embeddings/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/embeddings/add_1"
  input: "bert/embeddings/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/embeddings/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/embeddings/add_1"
  input: "bert/embeddings/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 2
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/embeddings/LayerNorm/moments/SquaredDifference"
  input: "bert/embeddings/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/embeddings/LayerNorm/moments/variance"
  input: "bert/embeddings/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/embeddings/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/embeddings/LayerNorm/batchnorm/Rsqrt"
  input: "bert/embeddings/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/embeddings/add_1"
  input: "bert/embeddings/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/embeddings/LayerNorm/moments/mean"
  input: "bert/embeddings/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/embeddings/LayerNorm/beta/read"
  input: "bert/embeddings/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/embeddings/LayerNorm/batchnorm/mul_1"
  input: "bert/embeddings/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/embeddings/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/embeddings/dropout/random_uniform/max"
  input: "bert/embeddings/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/embeddings/dropout/random_uniform/RandomUniform"
  input: "bert/embeddings/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/random_uniform"
  op: "Add"
  input: "bert/embeddings/dropout/random_uniform/mul"
  input: "bert/embeddings/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/add"
  op: "Add"
  input: "bert/embeddings/dropout/keep_prob"
  input: "bert/embeddings/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/Floor"
  op: "Floor"
  input: "bert/embeddings/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/div"
  op: "RealDiv"
  input: "bert/embeddings/LayerNorm/batchnorm/add_1"
  input: "bert/embeddings/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/embeddings/dropout/mul"
  op: "Mul"
  input: "bert/embeddings/dropout/div"
  input: "bert/embeddings/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\001\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/Reshape"
  op: "Reshape"
  input: "IteratorGetNext:1"
  input: "bert/encoder/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/Cast"
  op: "Cast"
  input: "bert/encoder/Reshape"
  attr {
    key: "DstT"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "SrcT"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "Truncate"
    value {
      b: false
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/ones/shape_as_tensor"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 3
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\001\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/ones/Const"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/ones"
  op: "Fill"
  input: "bert/encoder/ones/shape_as_tensor"
  input: "bert/encoder/ones/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/mul"
  op: "Mul"
  input: "bert/encoder/ones"
  input: "bert/encoder/Cast"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\377\377\377\377\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/Reshape_1"
  op: "Reshape"
  input: "bert/embeddings/dropout/mul"
  input: "bert/encoder/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/query/kernel"
  input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/query/bias"
  input: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/Reshape_1"
  input: "bert/encoder/layer_0/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/attention/self/query/MatMul"
  input: "bert/encoder/layer_0/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/key/kernel"
  input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/key/bias"
  input: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/Reshape_1"
  input: "bert/encoder/layer_0/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/attention/self/key/MatMul"
  input: "bert/encoder/layer_0/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/value/kernel"
  input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/self/value/bias"
  input: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/Reshape_1"
  input: "bert/encoder/layer_0/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/attention/self/value/MatMul"
  input: "bert/encoder/layer_0/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_0/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_0/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_0/attention/self/Reshape"
  input: "bert/encoder/layer_0/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_0/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_0/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_0/attention/self/Reshape_1"
  input: "bert/encoder/layer_0/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_0/attention/self/transpose"
  input: "bert/encoder/layer_0/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/MatMul"
  input: "bert/encoder/layer_0/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_0/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/attention/self/sub/x"
  input: "bert/encoder/layer_0/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/sub"
  input: "bert/encoder/layer_0/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/Mul"
  input: "bert/encoder/layer_0/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_0/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_0/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_0/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_0/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_0/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_0/attention/self/Softmax"
  input: "bert/encoder/layer_0/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/self/dropout/div"
  input: "bert/encoder/layer_0/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_0/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_0/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_0/attention/self/Reshape_2"
  input: "bert/encoder/layer_0/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_0/attention/self/dropout/mul"
  input: "bert/encoder/layer_0/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_0/attention/self/MatMul_1"
  input: "bert/encoder/layer_0/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_0/attention/self/transpose_3"
  input: "bert/encoder/layer_0/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/output/dense/kernel"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/output/dense/bias"
  input: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/attention/self/Reshape_3"
  input: "bert/encoder/layer_0/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/attention/output/dense/MatMul"
  input: "bert/encoder/layer_0/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_0/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_0/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_0/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_0/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_0/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/dropout/div"
  input: "bert/encoder/layer_0/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/dropout/mul"
  input: "bert/encoder/Reshape_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_0/attention/output/add"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_0/attention/output/add"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/add"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/intermediate/dense/kernel"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/intermediate/dense/bias"
  input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_0/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/intermediate/dense/MatMul"
  input: "bert/encoder/layer_0/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_0/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_0/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_0/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_0/intermediate/dense/add/x"
  input: "bert/encoder/layer_0/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/intermediate/dense/mul/x"
  input: "bert/encoder/layer_0/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_0/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_0/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/output/dense/kernel"
  input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_0/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/output/dense/bias"
  input: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_0/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/intermediate/dense/mul_1"
  input: "bert/encoder/layer_0/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_0/output/dense/MatMul"
  input: "bert/encoder/layer_0/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_0/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_0/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_0/output/dropout/keep_prob"
  input: "bert/encoder/layer_0/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_0/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_0/output/dense/BiasAdd"
  input: "bert/encoder/layer_0/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/output/dropout/div"
  input: "bert/encoder/layer_0/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/add"
  op: "Add"
  input: "bert/encoder/layer_0/output/dropout/mul"
  input: "bert/encoder/layer_0/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/output/LayerNorm/beta"
  input: "bert/encoder/layer_0/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_0/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_0/output/LayerNorm/gamma"
  input: "bert/encoder/layer_0/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_0/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_0/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_0/output/add"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_0/output/add"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_0/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_0/output/add"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_0/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_0/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/query/kernel"
  input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/query/bias"
  input: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_1/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/attention/self/query/MatMul"
  input: "bert/encoder/layer_1/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/key/kernel"
  input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/key/bias"
  input: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_1/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/attention/self/key/MatMul"
  input: "bert/encoder/layer_1/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/value/kernel"
  input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/self/value/bias"
  input: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_1/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/attention/self/value/MatMul"
  input: "bert/encoder/layer_1/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_1/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_1/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_1/attention/self/Reshape"
  input: "bert/encoder/layer_1/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_1/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_1/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_1/attention/self/Reshape_1"
  input: "bert/encoder/layer_1/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_1/attention/self/transpose"
  input: "bert/encoder/layer_1/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/MatMul"
  input: "bert/encoder/layer_1/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_1/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/attention/self/sub/x"
  input: "bert/encoder/layer_1/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/sub"
  input: "bert/encoder/layer_1/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/Mul"
  input: "bert/encoder/layer_1/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_1/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_1/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_1/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_1/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_1/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_1/attention/self/Softmax"
  input: "bert/encoder/layer_1/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/self/dropout/div"
  input: "bert/encoder/layer_1/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_1/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_1/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_1/attention/self/Reshape_2"
  input: "bert/encoder/layer_1/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_1/attention/self/dropout/mul"
  input: "bert/encoder/layer_1/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_1/attention/self/MatMul_1"
  input: "bert/encoder/layer_1/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_1/attention/self/transpose_3"
  input: "bert/encoder/layer_1/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/output/dense/kernel"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/output/dense/bias"
  input: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/attention/self/Reshape_3"
  input: "bert/encoder/layer_1/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/attention/output/dense/MatMul"
  input: "bert/encoder/layer_1/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_1/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_1/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_1/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_1/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_1/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/dropout/div"
  input: "bert/encoder/layer_1/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/dropout/mul"
  input: "bert/encoder/layer_0/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_1/attention/output/add"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_1/attention/output/add"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/add"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/intermediate/dense/kernel"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/intermediate/dense/bias"
  input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_1/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/intermediate/dense/MatMul"
  input: "bert/encoder/layer_1/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_1/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_1/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_1/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_1/intermediate/dense/add/x"
  input: "bert/encoder/layer_1/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/intermediate/dense/mul/x"
  input: "bert/encoder/layer_1/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_1/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_1/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/output/dense/kernel"
  input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_1/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/output/dense/bias"
  input: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_1/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/intermediate/dense/mul_1"
  input: "bert/encoder/layer_1/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_1/output/dense/MatMul"
  input: "bert/encoder/layer_1/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_1/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_1/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_1/output/dropout/keep_prob"
  input: "bert/encoder/layer_1/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_1/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_1/output/dense/BiasAdd"
  input: "bert/encoder/layer_1/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/output/dropout/div"
  input: "bert/encoder/layer_1/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/add"
  op: "Add"
  input: "bert/encoder/layer_1/output/dropout/mul"
  input: "bert/encoder/layer_1/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/output/LayerNorm/beta"
  input: "bert/encoder/layer_1/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_1/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_1/output/LayerNorm/gamma"
  input: "bert/encoder/layer_1/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_1/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_1/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_1/output/add"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_1/output/add"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_1/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_1/output/add"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_1/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_1/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/query/kernel"
  input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/query/bias"
  input: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_2/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/attention/self/query/MatMul"
  input: "bert/encoder/layer_2/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/key/kernel"
  input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/key/bias"
  input: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_2/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/attention/self/key/MatMul"
  input: "bert/encoder/layer_2/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/value/kernel"
  input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/self/value/bias"
  input: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_2/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/attention/self/value/MatMul"
  input: "bert/encoder/layer_2/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_2/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_2/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_2/attention/self/Reshape"
  input: "bert/encoder/layer_2/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_2/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_2/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_2/attention/self/Reshape_1"
  input: "bert/encoder/layer_2/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_2/attention/self/transpose"
  input: "bert/encoder/layer_2/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/MatMul"
  input: "bert/encoder/layer_2/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_2/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/attention/self/sub/x"
  input: "bert/encoder/layer_2/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/sub"
  input: "bert/encoder/layer_2/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/Mul"
  input: "bert/encoder/layer_2/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_2/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_2/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_2/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_2/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_2/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_2/attention/self/Softmax"
  input: "bert/encoder/layer_2/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/self/dropout/div"
  input: "bert/encoder/layer_2/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_2/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_2/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_2/attention/self/Reshape_2"
  input: "bert/encoder/layer_2/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_2/attention/self/dropout/mul"
  input: "bert/encoder/layer_2/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_2/attention/self/MatMul_1"
  input: "bert/encoder/layer_2/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_2/attention/self/transpose_3"
  input: "bert/encoder/layer_2/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/output/dense/kernel"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/output/dense/bias"
  input: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/attention/self/Reshape_3"
  input: "bert/encoder/layer_2/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/attention/output/dense/MatMul"
  input: "bert/encoder/layer_2/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_2/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_2/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_2/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_2/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_2/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/dropout/div"
  input: "bert/encoder/layer_2/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/dropout/mul"
  input: "bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_2/attention/output/add"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_2/attention/output/add"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/add"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/intermediate/dense/kernel"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/intermediate/dense/bias"
  input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_2/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/intermediate/dense/MatMul"
  input: "bert/encoder/layer_2/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_2/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_2/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_2/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_2/intermediate/dense/add/x"
  input: "bert/encoder/layer_2/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/intermediate/dense/mul/x"
  input: "bert/encoder/layer_2/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_2/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_2/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/output/dense/kernel"
  input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_2/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/output/dense/bias"
  input: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_2/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/intermediate/dense/mul_1"
  input: "bert/encoder/layer_2/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_2/output/dense/MatMul"
  input: "bert/encoder/layer_2/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_2/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_2/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_2/output/dropout/keep_prob"
  input: "bert/encoder/layer_2/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_2/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_2/output/dense/BiasAdd"
  input: "bert/encoder/layer_2/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/output/dropout/div"
  input: "bert/encoder/layer_2/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/add"
  op: "Add"
  input: "bert/encoder/layer_2/output/dropout/mul"
  input: "bert/encoder/layer_2/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/output/LayerNorm/beta"
  input: "bert/encoder/layer_2/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_2/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_2/output/LayerNorm/gamma"
  input: "bert/encoder/layer_2/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_2/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_2/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_2/output/add"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_2/output/add"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_2/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_2/output/add"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_2/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_2/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/query/kernel"
  input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/query/bias"
  input: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_3/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/attention/self/query/MatMul"
  input: "bert/encoder/layer_3/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/key/kernel"
  input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/key/bias"
  input: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_3/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/attention/self/key/MatMul"
  input: "bert/encoder/layer_3/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/value/kernel"
  input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/self/value/bias"
  input: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_3/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/attention/self/value/MatMul"
  input: "bert/encoder/layer_3/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_3/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_3/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_3/attention/self/Reshape"
  input: "bert/encoder/layer_3/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_3/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_3/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_3/attention/self/Reshape_1"
  input: "bert/encoder/layer_3/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_3/attention/self/transpose"
  input: "bert/encoder/layer_3/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/MatMul"
  input: "bert/encoder/layer_3/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_3/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/attention/self/sub/x"
  input: "bert/encoder/layer_3/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/sub"
  input: "bert/encoder/layer_3/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/Mul"
  input: "bert/encoder/layer_3/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_3/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_3/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_3/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_3/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_3/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_3/attention/self/Softmax"
  input: "bert/encoder/layer_3/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/self/dropout/div"
  input: "bert/encoder/layer_3/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_3/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_3/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_3/attention/self/Reshape_2"
  input: "bert/encoder/layer_3/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_3/attention/self/dropout/mul"
  input: "bert/encoder/layer_3/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_3/attention/self/MatMul_1"
  input: "bert/encoder/layer_3/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_3/attention/self/transpose_3"
  input: "bert/encoder/layer_3/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/output/dense/kernel"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/output/dense/bias"
  input: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/attention/self/Reshape_3"
  input: "bert/encoder/layer_3/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/attention/output/dense/MatMul"
  input: "bert/encoder/layer_3/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_3/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_3/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_3/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_3/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_3/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/dropout/div"
  input: "bert/encoder/layer_3/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/dropout/mul"
  input: "bert/encoder/layer_2/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_3/attention/output/add"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_3/attention/output/add"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/add"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/intermediate/dense/kernel"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/intermediate/dense/bias"
  input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_3/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/intermediate/dense/MatMul"
  input: "bert/encoder/layer_3/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_3/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_3/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_3/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_3/intermediate/dense/add/x"
  input: "bert/encoder/layer_3/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/intermediate/dense/mul/x"
  input: "bert/encoder/layer_3/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_3/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_3/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/output/dense/kernel"
  input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_3/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/output/dense/bias"
  input: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_3/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/intermediate/dense/mul_1"
  input: "bert/encoder/layer_3/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_3/output/dense/MatMul"
  input: "bert/encoder/layer_3/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_3/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_3/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_3/output/dropout/keep_prob"
  input: "bert/encoder/layer_3/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_3/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_3/output/dense/BiasAdd"
  input: "bert/encoder/layer_3/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/output/dropout/div"
  input: "bert/encoder/layer_3/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/add"
  op: "Add"
  input: "bert/encoder/layer_3/output/dropout/mul"
  input: "bert/encoder/layer_3/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/output/LayerNorm/beta"
  input: "bert/encoder/layer_3/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_3/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_3/output/LayerNorm/gamma"
  input: "bert/encoder/layer_3/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_3/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_3/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_3/output/add"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_3/output/add"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_3/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_3/output/add"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_3/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_3/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/query/kernel"
  input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/query/bias"
  input: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_4/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/attention/self/query/MatMul"
  input: "bert/encoder/layer_4/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/key/kernel"
  input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/key/bias"
  input: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_4/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/attention/self/key/MatMul"
  input: "bert/encoder/layer_4/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/value/kernel"
  input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/self/value/bias"
  input: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_4/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/attention/self/value/MatMul"
  input: "bert/encoder/layer_4/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_4/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_4/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_4/attention/self/Reshape"
  input: "bert/encoder/layer_4/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_4/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_4/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_4/attention/self/Reshape_1"
  input: "bert/encoder/layer_4/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_4/attention/self/transpose"
  input: "bert/encoder/layer_4/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/MatMul"
  input: "bert/encoder/layer_4/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_4/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/attention/self/sub/x"
  input: "bert/encoder/layer_4/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/sub"
  input: "bert/encoder/layer_4/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/Mul"
  input: "bert/encoder/layer_4/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_4/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_4/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_4/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_4/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_4/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_4/attention/self/Softmax"
  input: "bert/encoder/layer_4/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/self/dropout/div"
  input: "bert/encoder/layer_4/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_4/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_4/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_4/attention/self/Reshape_2"
  input: "bert/encoder/layer_4/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_4/attention/self/dropout/mul"
  input: "bert/encoder/layer_4/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_4/attention/self/MatMul_1"
  input: "bert/encoder/layer_4/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_4/attention/self/transpose_3"
  input: "bert/encoder/layer_4/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/output/dense/kernel"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/output/dense/bias"
  input: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/attention/self/Reshape_3"
  input: "bert/encoder/layer_4/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/attention/output/dense/MatMul"
  input: "bert/encoder/layer_4/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_4/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_4/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_4/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_4/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_4/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/dropout/div"
  input: "bert/encoder/layer_4/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/dropout/mul"
  input: "bert/encoder/layer_3/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_4/attention/output/add"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_4/attention/output/add"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/add"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/intermediate/dense/kernel"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/intermediate/dense/bias"
  input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_4/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/intermediate/dense/MatMul"
  input: "bert/encoder/layer_4/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_4/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_4/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_4/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_4/intermediate/dense/add/x"
  input: "bert/encoder/layer_4/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/intermediate/dense/mul/x"
  input: "bert/encoder/layer_4/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_4/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_4/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/output/dense/kernel"
  input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_4/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/output/dense/bias"
  input: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_4/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/intermediate/dense/mul_1"
  input: "bert/encoder/layer_4/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_4/output/dense/MatMul"
  input: "bert/encoder/layer_4/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_4/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_4/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_4/output/dropout/keep_prob"
  input: "bert/encoder/layer_4/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_4/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_4/output/dense/BiasAdd"
  input: "bert/encoder/layer_4/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/output/dropout/div"
  input: "bert/encoder/layer_4/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/add"
  op: "Add"
  input: "bert/encoder/layer_4/output/dropout/mul"
  input: "bert/encoder/layer_4/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/output/LayerNorm/beta"
  input: "bert/encoder/layer_4/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_4/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_4/output/LayerNorm/gamma"
  input: "bert/encoder/layer_4/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_4/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_4/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_4/output/add"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_4/output/add"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_4/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_4/output/add"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_4/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_4/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/query/kernel"
  input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/query/bias"
  input: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_5/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/attention/self/query/MatMul"
  input: "bert/encoder/layer_5/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/key/kernel"
  input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/key/bias"
  input: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_5/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/attention/self/key/MatMul"
  input: "bert/encoder/layer_5/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/value/kernel"
  input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/self/value/bias"
  input: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_5/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/attention/self/value/MatMul"
  input: "bert/encoder/layer_5/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_5/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_5/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_5/attention/self/Reshape"
  input: "bert/encoder/layer_5/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_5/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_5/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_5/attention/self/Reshape_1"
  input: "bert/encoder/layer_5/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_5/attention/self/transpose"
  input: "bert/encoder/layer_5/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/MatMul"
  input: "bert/encoder/layer_5/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_5/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/attention/self/sub/x"
  input: "bert/encoder/layer_5/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/sub"
  input: "bert/encoder/layer_5/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/Mul"
  input: "bert/encoder/layer_5/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_5/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_5/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_5/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_5/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_5/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_5/attention/self/Softmax"
  input: "bert/encoder/layer_5/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/self/dropout/div"
  input: "bert/encoder/layer_5/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_5/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_5/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_5/attention/self/Reshape_2"
  input: "bert/encoder/layer_5/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_5/attention/self/dropout/mul"
  input: "bert/encoder/layer_5/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_5/attention/self/MatMul_1"
  input: "bert/encoder/layer_5/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_5/attention/self/transpose_3"
  input: "bert/encoder/layer_5/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/output/dense/kernel"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/output/dense/bias"
  input: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/attention/self/Reshape_3"
  input: "bert/encoder/layer_5/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/attention/output/dense/MatMul"
  input: "bert/encoder/layer_5/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_5/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_5/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_5/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_5/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_5/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/dropout/div"
  input: "bert/encoder/layer_5/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/dropout/mul"
  input: "bert/encoder/layer_4/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_5/attention/output/add"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_5/attention/output/add"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/add"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/intermediate/dense/kernel"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/intermediate/dense/bias"
  input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_5/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/intermediate/dense/MatMul"
  input: "bert/encoder/layer_5/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_5/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_5/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_5/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_5/intermediate/dense/add/x"
  input: "bert/encoder/layer_5/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/intermediate/dense/mul/x"
  input: "bert/encoder/layer_5/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_5/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_5/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/output/dense/kernel"
  input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_5/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/output/dense/bias"
  input: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_5/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/intermediate/dense/mul_1"
  input: "bert/encoder/layer_5/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_5/output/dense/MatMul"
  input: "bert/encoder/layer_5/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_5/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_5/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_5/output/dropout/keep_prob"
  input: "bert/encoder/layer_5/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_5/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_5/output/dense/BiasAdd"
  input: "bert/encoder/layer_5/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/output/dropout/div"
  input: "bert/encoder/layer_5/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/add"
  op: "Add"
  input: "bert/encoder/layer_5/output/dropout/mul"
  input: "bert/encoder/layer_5/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/output/LayerNorm/beta"
  input: "bert/encoder/layer_5/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_5/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_5/output/LayerNorm/gamma"
  input: "bert/encoder/layer_5/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_5/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_5/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_5/output/add"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_5/output/add"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_5/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_5/output/add"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_5/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_5/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/query/kernel"
  input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/query/bias"
  input: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_6/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/attention/self/query/MatMul"
  input: "bert/encoder/layer_6/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/key/kernel"
  input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/key/bias"
  input: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_6/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/attention/self/key/MatMul"
  input: "bert/encoder/layer_6/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/value/kernel"
  input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/self/value/bias"
  input: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_6/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/attention/self/value/MatMul"
  input: "bert/encoder/layer_6/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_6/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_6/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_6/attention/self/Reshape"
  input: "bert/encoder/layer_6/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_6/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_6/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_6/attention/self/Reshape_1"
  input: "bert/encoder/layer_6/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_6/attention/self/transpose"
  input: "bert/encoder/layer_6/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/MatMul"
  input: "bert/encoder/layer_6/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_6/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/attention/self/sub/x"
  input: "bert/encoder/layer_6/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/sub"
  input: "bert/encoder/layer_6/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/Mul"
  input: "bert/encoder/layer_6/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_6/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_6/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_6/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_6/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_6/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_6/attention/self/Softmax"
  input: "bert/encoder/layer_6/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/self/dropout/div"
  input: "bert/encoder/layer_6/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_6/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_6/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_6/attention/self/Reshape_2"
  input: "bert/encoder/layer_6/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_6/attention/self/dropout/mul"
  input: "bert/encoder/layer_6/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_6/attention/self/MatMul_1"
  input: "bert/encoder/layer_6/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_6/attention/self/transpose_3"
  input: "bert/encoder/layer_6/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/output/dense/kernel"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/output/dense/bias"
  input: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/attention/self/Reshape_3"
  input: "bert/encoder/layer_6/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/attention/output/dense/MatMul"
  input: "bert/encoder/layer_6/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_6/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_6/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_6/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_6/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_6/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/dropout/div"
  input: "bert/encoder/layer_6/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/dropout/mul"
  input: "bert/encoder/layer_5/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_6/attention/output/add"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_6/attention/output/add"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/add"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/intermediate/dense/kernel"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/intermediate/dense/bias"
  input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_6/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/intermediate/dense/MatMul"
  input: "bert/encoder/layer_6/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_6/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_6/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_6/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_6/intermediate/dense/add/x"
  input: "bert/encoder/layer_6/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/intermediate/dense/mul/x"
  input: "bert/encoder/layer_6/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_6/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_6/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/output/dense/kernel"
  input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_6/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/output/dense/bias"
  input: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_6/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/intermediate/dense/mul_1"
  input: "bert/encoder/layer_6/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_6/output/dense/MatMul"
  input: "bert/encoder/layer_6/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_6/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_6/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_6/output/dropout/keep_prob"
  input: "bert/encoder/layer_6/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_6/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_6/output/dense/BiasAdd"
  input: "bert/encoder/layer_6/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/output/dropout/div"
  input: "bert/encoder/layer_6/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/add"
  op: "Add"
  input: "bert/encoder/layer_6/output/dropout/mul"
  input: "bert/encoder/layer_6/attention/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/output/LayerNorm/beta"
  input: "bert/encoder/layer_6/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_6/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_6/output/LayerNorm/gamma"
  input: "bert/encoder/layer_6/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_6/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_6/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_6/output/add"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_6/output/add"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_6/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_6/output/add"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_6/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_6/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/query/kernel"
  input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/query/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/query/bias"
  input: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/query/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/query/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_7/attention/self/query/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/query/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_7/attention/self/query/MatMul"
  input: "bert/encoder/layer_7/attention/self/query/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/key/kernel"
  input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/key/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/key/bias"
  input: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/key/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/key/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_7/attention/self/key/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/key/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_7/attention/self/key/MatMul"
  input: "bert/encoder/layer_7/attention/self/key/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/value/kernel"
  input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/value/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/self/value/bias"
  input: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/self/value/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/self/value/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_7/attention/self/value/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/value/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_7/attention/self/value/MatMul"
  input: "bert/encoder/layer_7/attention/self/value/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape"
  op: "Reshape"
  input: "bert/encoder/layer_7/attention/self/query/BiasAdd"
  input: "bert/encoder/layer_7/attention/self/Reshape/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose"
  op: "Transpose"
  input: "bert/encoder/layer_7/attention/self/Reshape"
  input: "bert/encoder/layer_7/attention/self/transpose/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_1/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_1"
  op: "Reshape"
  input: "bert/encoder/layer_7/attention/self/key/BiasAdd"
  input: "bert/encoder/layer_7/attention/self/Reshape_1/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_1/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_1"
  op: "Transpose"
  input: "bert/encoder/layer_7/attention/self/Reshape_1"
  input: "bert/encoder/layer_7/attention/self/transpose_1/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/MatMul"
  op: "BatchMatMul"
  input: "bert/encoder/layer_7/attention/self/transpose"
  input: "bert/encoder/layer_7/attention/self/transpose_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Mul/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.125
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/MatMul"
  input: "bert/encoder/layer_7/attention/self/Mul/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/ExpandDims/dim"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/ExpandDims"
  op: "ExpandDims"
  input: "bert/encoder/mul"
  input: "bert/encoder/layer_7/attention/self/ExpandDims/dim"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tdim"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/sub/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/sub"
  op: "Sub"
  input: "bert/encoder/layer_7/attention/self/sub/x"
  input: "bert/encoder/layer_7/attention/self/ExpandDims"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/mul_1/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: -10000.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/sub"
  input: "bert/encoder/layer_7/attention/self/mul_1/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 1
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/add"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/Mul"
  input: "bert/encoder/layer_7/attention/self/mul_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Softmax"
  op: "Softmax"
  input: "bert/encoder/layer_7/attention/self/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\014\000\000\000\200\000\000\000\200\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_7/attention/self/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/max"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/mul"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_7/attention/self/dropout/keep_prob"
  input: "bert/encoder/layer_7/attention/self/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_7/attention/self/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_7/attention/self/Softmax"
  input: "bert/encoder/layer_7/attention/self/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/self/dropout/div"
  input: "bert/encoder/layer_7/attention/self/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 128
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_2/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: " \000\000\000\200\000\000\000\014\000\000\000@\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_2"
  op: "Reshape"
  input: "bert/encoder/layer_7/attention/self/value/BiasAdd"
  input: "bert/encoder/layer_7/attention/self/Reshape_2/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_2/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_2"
  op: "Transpose"
  input: "bert/encoder/layer_7/attention/self/Reshape_2"
  input: "bert/encoder/layer_7/attention/self/transpose_2/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/MatMul_1"
  op: "BatchMatMul"
  input: "bert/encoder/layer_7/attention/self/dropout/mul"
  input: "bert/encoder/layer_7/attention/self/transpose_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 12
          }
          dim {
            size: 128
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
  attr {
    key: "adj_x"
    value {
      b: false
    }
  }
  attr {
    key: "adj_y"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_3/perm"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 4
          }
        }
        tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/transpose_3"
  op: "Transpose"
  input: "bert/encoder/layer_7/attention/self/MatMul_1"
  input: "bert/encoder/layer_7/attention/self/transpose_3/perm"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tperm"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 32
          }
          dim {
            size: 128
          }
          dim {
            size: 12
          }
          dim {
            size: 64
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_3/shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/self/Reshape_3"
  op: "Reshape"
  input: "bert/encoder/layer_7/attention/self/transpose_3"
  input: "bert/encoder/layer_7/attention/self/Reshape_3/shape"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tshape"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/output/dense/kernel"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/output/dense/bias"
  input: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_7/attention/self/Reshape_3"
  input: "bert/encoder/layer_7/attention/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_7/attention/output/dense/MatMul"
  input: "bert/encoder/layer_7/attention/output/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/keep_prob"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.8999999761581421
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/Shape"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\020\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/max"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  op: "RandomUniform"
  input: "bert/encoder/layer_7/attention/output/dropout/Shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/sub"
  op: "Sub"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/max"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/random_uniform"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/mul"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/min"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/add"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/dropout/keep_prob"
  input: "bert/encoder/layer_7/attention/output/dropout/random_uniform"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/Floor"
  op: "Floor"
  input: "bert/encoder/layer_7/attention/output/dropout/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/div"
  op: "RealDiv"
  input: "bert/encoder/layer_7/attention/output/dense/BiasAdd"
  input: "bert/encoder/layer_7/attention/output/dropout/keep_prob"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/dropout/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/dropout/div"
  input: "bert/encoder/layer_7/attention/output/dropout/Floor"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/add"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/dropout/mul"
  input: "bert/encoder/layer_6/output/LayerNorm/batchnorm/add_1"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/beta"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/beta"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/beta/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/beta"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/beta"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/Initializer/ones"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read"
  op: "Identity"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/attention/output/LayerNorm/gamma"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  op: "Mean"
  input: "bert/encoder/layer_7/attention/output/add"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  op: "StopGradient"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  op: "SquaredDifference"
  input: "bert/encoder/layer_7/attention/output/add"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/StopGradient"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 1
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  op: "Mean"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/SquaredDifference"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance/reduction_indices"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "Tidx"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "keep_dims"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 9.999999960041972e-13
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/variance"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add/y"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  op: "Rsqrt"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 1
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/Rsqrt"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/gamma/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/add"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  op: "Mul"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/moments/mean"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  op: "Sub"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/beta/read"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_2"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  op: "Add"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/mul_1"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/sub"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\003\000\000\000\014\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/intermediate/dense/kernel"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/intermediate/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 1
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 1
          }
        }
        int_val: 3072
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  op: "Fill"
  input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor"
  input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "index_type"
    value {
      type: DT_INT32
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/intermediate/dense/bias"
  input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/intermediate/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/intermediate/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_7/attention/output/LayerNorm/batchnorm/add_1"
  input: "bert/encoder/layer_7/intermediate/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"
    value {
      b: false
    }
  }
  attr {
    key: "transpose_b"
    value {
      b: false
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/BiasAdd"
  op: "BiasAdd"
  input: "bert/encoder/layer_7/intermediate/dense/MatMul"
  input: "bert/encoder/layer_7/intermediate/dense/bias/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
  attr {
    key: "data_format"
    value {
      s: "NHWC"
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/Sqrt/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 2.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/Sqrt"
  op: "Sqrt"
  input: "bert/encoder/layer_7/intermediate/dense/Sqrt/x"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/truediv"
  op: "RealDiv"
  input: "bert/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_7/intermediate/dense/Sqrt"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/Erf"
  op: "Erf"
  input: "bert/encoder/layer_7/intermediate/dense/truediv"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/add/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 1.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/add"
  op: "Add"
  input: "bert/encoder/layer_7/intermediate/dense/add/x"
  input: "bert/encoder/layer_7/intermediate/dense/Erf"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/mul/x"
  op: "Const"
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.5
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/intermediate/dense/mul/x"
  input: "bert/encoder/layer_7/intermediate/dense/add"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/intermediate/dense/mul_1"
  op: "Mul"
  input: "bert/encoder/layer_7/intermediate/dense/BiasAdd"
  input: "bert/encoder/layer_7/intermediate/dense/mul"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 3072
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 2
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_INT32
        tensor_shape {
          dim {
            size: 2
          }
        }
        tensor_content: "\000\014\000\000\000\003\000\000"
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
        }
        float_val: 0.019999999552965164
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  op: "TruncatedNormal"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape"
  attr {
    key: "T"
    value {
      type: DT_INT32
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "seed"
    value {
      i: 0
    }
  }
  attr {
    key: "seed2"
    value {
      i: 0
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  op: "Mul"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  op: "Add"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 3072
        }
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/output/dense/kernel"
  input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/kernel/read"
  op: "Identity"
  input: "bert/encoder/layer_7/output/dense/kernel"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/kernel"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 3072
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros"
  op: "Const"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "value"
    value {
      tensor {
        dtype: DT_FLOAT
        tensor_shape {
          dim {
            size: 768
          }
        }
        float_val: 0.0
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/bias"
  op: "VariableV2"
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "container"
    value {
      s: ""
    }
  }
  attr {
    key: "dtype"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "shape"
    value {
      shape {
        dim {
          size: 768
        }
      }
    }
  }
  attr {
    key: "shared_name"
    value {
      s: ""
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/bias/Assign"
  op: "Assign"
  input: "bert/encoder/layer_7/output/dense/bias"
  input: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "use_locking"
    value {
      b: true
    }
  }
  attr {
    key: "validate_shape"
    value {
      b: true
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/bias/read"
  op: "Identity"
  input: "bert/encoder/layer_7/output/dense/bias"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_class"
    value {
      list {
        s: "loc:@bert/encoder/layer_7/output/dense/bias"
      }
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 768
          }
        }
      }
    }
  }
}
node {
  name: "bert/encoder/layer_7/output/dense/MatMul"
  op: "MatMul"
  input: "bert/encoder/layer_7/intermediate/dense/mul_1"
  input: "bert/encoder/layer_7/output/dense/kernel/read"
  attr {
    key: "T"
    value {
      type: DT_FLOAT
    }
  }
  attr {
    key: "_output_shapes"
    value {
      list {
        shape {
          dim {
            size: 4096
          }
          dim {
            size: 768
          }
        }
      }
    }
  }
  attr {
    key: "transpose_a"